2024-11-17 12:45:05,424 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-17 12:45:05,442 main DEBUG Took 0.014701 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-11-17 12:45:05,442 main DEBUG PluginManager 'Core' found 129 plugins
2024-11-17 12:45:05,443 main DEBUG PluginManager 'Level' found 0 plugins
2024-11-17 12:45:05,444 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-11-17 12:45:05,446 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-17 12:45:05,455 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-11-17 12:45:05,467 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 12:45:05,468 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-17 12:45:05,469 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 12:45:05,469 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-17 12:45:05,470 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 12:45:05,470 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-17 12:45:05,471 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 12:45:05,471 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-17 12:45:05,472 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 12:45:05,472 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-17 12:45:05,473 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 12:45:05,473 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-17 12:45:05,474 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 12:45:05,474 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-17 12:45:05,474 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 12:45:05,475 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-17 12:45:05,475 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 12:45:05,475 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-17 12:45:05,476 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 12:45:05,476 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-17 12:45:05,477 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 12:45:05,477 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-17 12:45:05,477 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 12:45:05,478 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-17 12:45:05,478 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 12:45:05,478 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-11-17 12:45:05,480 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-17 12:45:05,481 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-11-17 12:45:05,483 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-11-17 12:45:05,483 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-11-17 12:45:05,484 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-11-17 12:45:05,485 main DEBUG PluginManager 'Converter' found 47 plugins
2024-11-17 12:45:05,493 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-11-17 12:45:05,495 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-11-17 12:45:05,497 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-11-17 12:45:05,497 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-11-17 12:45:05,498 main DEBUG createAppenders(={Console})
2024-11-17 12:45:05,498 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized
2024-11-17 12:45:05,499 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-17 12:45:05,499 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK.
2024-11-17 12:45:05,499 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-11-17 12:45:05,500 main DEBUG OutputStream closed
2024-11-17 12:45:05,500 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-11-17 12:45:05,500 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-11-17 12:45:05,500 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK
2024-11-17 12:45:05,571 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-11-17 12:45:05,573 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-11-17 12:45:05,574 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-11-17 12:45:05,575 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-11-17 12:45:05,576 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-11-17 12:45:05,576 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-11-17 12:45:05,576 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-11-17 12:45:05,577 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-11-17 12:45:05,577 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-11-17 12:45:05,577 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-11-17 12:45:05,578 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-11-17 12:45:05,578 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-11-17 12:45:05,578 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-11-17 12:45:05,579 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-11-17 12:45:05,579 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-11-17 12:45:05,579 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-11-17 12:45:05,580 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-11-17 12:45:05,580 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-11-17 12:45:05,583 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-17 12:45:05,583 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null
2024-11-17 12:45:05,583 main DEBUG Shutdown hook enabled. Registering a new one.
2024-11-17 12:45:05,584 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK.
2024-11-17T12:45:05,812 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1
2024-11-17 12:45:05,815 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-11-17 12:45:05,815 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-17T12:45:05,823 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins
2024-11-17T12:45:05,856 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=248, ProcessCount=11, AvailableMemoryMB=6722
2024-11-17T12:45:05,859 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-17T12:45:05,874 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/cluster_1ccbeaaa-4148-7ce0-570b-22872a7cb186, deleteOnExit=true
2024-11-17T12:45:05,874 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-17T12:45:05,875 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/test.cache.data in system properties and HBase conf
2024-11-17T12:45:05,876 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/hadoop.tmp.dir in system properties and HBase conf
2024-11-17T12:45:05,876 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/hadoop.log.dir in system properties and HBase conf
2024-11-17T12:45:05,877 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-17T12:45:05,877 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-17T12:45:05,877 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-17T12:45:05,952 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-11-17T12:45:06,036 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-17T12:45:06,040 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-17T12:45:06,041 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-17T12:45:06,042 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-17T12:45:06,042 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T12:45:06,043 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-17T12:45:06,043 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-17T12:45:06,044 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T12:45:06,044 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T12:45:06,045 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-17T12:45:06,045 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/nfs.dump.dir in system properties and HBase conf
2024-11-17T12:45:06,046 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/java.io.tmpdir in system properties and HBase conf
2024-11-17T12:45:06,046 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T12:45:06,047 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-17T12:45:06,047 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-17T12:45:06,531 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T12:45:07,053 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-11-17T12:45:07,123 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2363ms to org.eclipse.jetty.util.log.Slf4jLog
2024-11-17T12:45:07,189 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:45:07,249 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:45:07,267 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:45:07,267 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:45:07,269 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-17T12:45:07,279 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:45:07,282 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:45:07,283 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:45:07,473 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@735fa16a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/java.io.tmpdir/jetty-localhost-34359-hadoop-hdfs-3_4_1-tests_jar-_-any-2052685611002006920/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T12:45:07,479 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:34359}
2024-11-17T12:45:07,479 INFO  [Time-limited test {}] server.Server(415): Started @2721ms
2024-11-17T12:45:07,502 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T12:45:07,950 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:45:07,957 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:45:07,958 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:45:07,958 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:45:07,959 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-17T12:45:07,959 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:45:07,960 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:45:08,056 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7b07d1ba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/java.io.tmpdir/jetty-localhost-46717-hadoop-hdfs-3_4_1-tests_jar-_-any-653859357378857224/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:45:08,056 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:46717}
2024-11-17T12:45:08,057 INFO  [Time-limited test {}] server.Server(415): Started @3298ms
2024-11-17T12:45:08,105 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T12:45:08,208 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:45:08,214 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:45:08,215 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:45:08,215 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:45:08,216 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T12:45:08,217 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:45:08,217 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:45:08,317 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bf97579{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/java.io.tmpdir/jetty-localhost-42877-hadoop-hdfs-3_4_1-tests_jar-_-any-16376823843951058738/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:45:08,318 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:42877}
2024-11-17T12:45:08,318 INFO  [Time-limited test {}] server.Server(415): Started @3560ms
2024-11-17T12:45:08,321 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T12:45:09,138 WARN  [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/cluster_1ccbeaaa-4148-7ce0-570b-22872a7cb186/data/data3/current/BP-362034389-172.17.0.2-1731847506610/current, will proceed with Du for space computation calculation, 
2024-11-17T12:45:09,138 WARN  [Thread-101 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/cluster_1ccbeaaa-4148-7ce0-570b-22872a7cb186/data/data2/current/BP-362034389-172.17.0.2-1731847506610/current, will proceed with Du for space computation calculation, 
2024-11-17T12:45:09,138 WARN  [Thread-100 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/cluster_1ccbeaaa-4148-7ce0-570b-22872a7cb186/data/data4/current/BP-362034389-172.17.0.2-1731847506610/current, will proceed with Du for space computation calculation, 
2024-11-17T12:45:09,138 WARN  [Thread-99 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/cluster_1ccbeaaa-4148-7ce0-570b-22872a7cb186/data/data1/current/BP-362034389-172.17.0.2-1731847506610/current, will proceed with Du for space computation calculation, 
2024-11-17T12:45:09,165 WARN  [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T12:45:09,165 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T12:45:09,212 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x246c4b162cd892d4 with lease ID 0xa54c93d03c8978ff: Processing first storage report for DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf from datanode DatanodeRegistration(127.0.0.1:43907, datanodeUuid=4648dfcf-5b9f-4a8c-84a5-b40fdbefd53c, infoPort=40369, infoSecurePort=0, ipcPort=32807, storageInfo=lv=-57;cid=testClusterID;nsid=44600911;c=1731847506610)
2024-11-17T12:45:09,213 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x246c4b162cd892d4 with lease ID 0xa54c93d03c8978ff: from storage DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf node DatanodeRegistration(127.0.0.1:43907, datanodeUuid=4648dfcf-5b9f-4a8c-84a5-b40fdbefd53c, infoPort=40369, infoSecurePort=0, ipcPort=32807, storageInfo=lv=-57;cid=testClusterID;nsid=44600911;c=1731847506610), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-17T12:45:09,213 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7bb8a7d7e9d8e3c with lease ID 0xa54c93d03c897900: Processing first storage report for DS-0952eab7-880d-498a-bdfe-f601e6454cfd from datanode DatanodeRegistration(127.0.0.1:36729, datanodeUuid=6c2e2e5a-ea18-4b82-b022-c156cf4c2479, infoPort=38107, infoSecurePort=0, ipcPort=46829, storageInfo=lv=-57;cid=testClusterID;nsid=44600911;c=1731847506610)
2024-11-17T12:45:09,213 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7bb8a7d7e9d8e3c with lease ID 0xa54c93d03c897900: from storage DS-0952eab7-880d-498a-bdfe-f601e6454cfd node DatanodeRegistration(127.0.0.1:36729, datanodeUuid=6c2e2e5a-ea18-4b82-b022-c156cf4c2479, infoPort=38107, infoSecurePort=0, ipcPort=46829, storageInfo=lv=-57;cid=testClusterID;nsid=44600911;c=1731847506610), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:45:09,214 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x246c4b162cd892d4 with lease ID 0xa54c93d03c8978ff: Processing first storage report for DS-65f7803c-9ad1-48e1-8b90-86b90ab14ced from datanode DatanodeRegistration(127.0.0.1:43907, datanodeUuid=4648dfcf-5b9f-4a8c-84a5-b40fdbefd53c, infoPort=40369, infoSecurePort=0, ipcPort=32807, storageInfo=lv=-57;cid=testClusterID;nsid=44600911;c=1731847506610)
2024-11-17T12:45:09,214 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x246c4b162cd892d4 with lease ID 0xa54c93d03c8978ff: from storage DS-65f7803c-9ad1-48e1-8b90-86b90ab14ced node DatanodeRegistration(127.0.0.1:43907, datanodeUuid=4648dfcf-5b9f-4a8c-84a5-b40fdbefd53c, infoPort=40369, infoSecurePort=0, ipcPort=32807, storageInfo=lv=-57;cid=testClusterID;nsid=44600911;c=1731847506610), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:45:09,214 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7bb8a7d7e9d8e3c with lease ID 0xa54c93d03c897900: Processing first storage report for DS-e742ec29-730a-43d6-a51a-a5df0bd23b51 from datanode DatanodeRegistration(127.0.0.1:36729, datanodeUuid=6c2e2e5a-ea18-4b82-b022-c156cf4c2479, infoPort=38107, infoSecurePort=0, ipcPort=46829, storageInfo=lv=-57;cid=testClusterID;nsid=44600911;c=1731847506610)
2024-11-17T12:45:09,214 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7bb8a7d7e9d8e3c with lease ID 0xa54c93d03c897900: from storage DS-e742ec29-730a-43d6-a51a-a5df0bd23b51 node DatanodeRegistration(127.0.0.1:36729, datanodeUuid=6c2e2e5a-ea18-4b82-b022-c156cf4c2479, infoPort=38107, infoSecurePort=0, ipcPort=46829, storageInfo=lv=-57;cid=testClusterID;nsid=44600911;c=1731847506610), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:45:09,223 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1
2024-11-17T12:45:09,287 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/cluster_1ccbeaaa-4148-7ce0-570b-22872a7cb186/zookeeper_0, clientPort=55241, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/cluster_1ccbeaaa-4148-7ce0-570b-22872a7cb186/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/cluster_1ccbeaaa-4148-7ce0-570b-22872a7cb186/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-17T12:45:09,296 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55241
2024-11-17T12:45:09,309 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:45:09,314 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:45:09,549 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741825_1001 (size=7)
2024-11-17T12:45:09,550 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741825_1001 (size=7)
2024-11-17T12:45:09,959 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f with version=8
2024-11-17T12:45:09,960 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/hbase-staging
2024-11-17T12:45:10,039 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-11-17T12:45:10,262 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/0d751a622496:0 server-side Connection retries=45
2024-11-17T12:45:10,271 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:45:10,272 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T12:45:10,276 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T12:45:10,277 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:45:10,277 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T12:45:10,422 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-17T12:45:10,484 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-11-17T12:45:10,494 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-11-17T12:45:10,498 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T12:45:10,528 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 20922 (auto-detected)
2024-11-17T12:45:10,529 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-11-17T12:45:10,552 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42805
2024-11-17T12:45:10,577 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42805 connecting to ZooKeeper ensemble=127.0.0.1:55241
2024-11-17T12:45:10,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:428050x0, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T12:45:10,712 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42805-0x1014a1d67db0000 connected
2024-11-17T12:45:10,803 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:45:10,806 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:45:10,818 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:45:10,822 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f, hbase.cluster.distributed=false
2024-11-17T12:45:10,853 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T12:45:10,861 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42805
2024-11-17T12:45:10,862 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42805
2024-11-17T12:45:10,867 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42805
2024-11-17T12:45:10,872 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42805
2024-11-17T12:45:10,873 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42805
2024-11-17T12:45:11,005 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/0d751a622496:0 server-side Connection retries=45
2024-11-17T12:45:11,006 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:45:11,007 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T12:45:11,007 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T12:45:11,007 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:45:11,007 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T12:45:11,011 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-17T12:45:11,014 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T12:45:11,015 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39415
2024-11-17T12:45:11,017 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39415 connecting to ZooKeeper ensemble=127.0.0.1:55241
2024-11-17T12:45:11,018 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:45:11,023 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:45:11,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:394150x0, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T12:45:11,058 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:394150x0, quorum=127.0.0.1:55241, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:45:11,058 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39415-0x1014a1d67db0001 connected
2024-11-17T12:45:11,062 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-17T12:45:11,070 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-17T12:45:11,073 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39415-0x1014a1d67db0001, quorum=127.0.0.1:55241, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-17T12:45:11,078 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39415-0x1014a1d67db0001, quorum=127.0.0.1:55241, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T12:45:11,079 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39415
2024-11-17T12:45:11,080 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39415
2024-11-17T12:45:11,081 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39415
2024-11-17T12:45:11,082 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39415
2024-11-17T12:45:11,082 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39415
2024-11-17T12:45:11,100 DEBUG [M:0;0d751a622496:42805 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0d751a622496:42805
2024-11-17T12:45:11,101 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0d751a622496,42805,1731847510113
2024-11-17T12:45:11,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:45:11,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x1014a1d67db0001, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:45:11,136 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0d751a622496,42805,1731847510113
2024-11-17T12:45:11,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:45:11,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x1014a1d67db0001, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-17T12:45:11,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x1014a1d67db0001, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:45:11,166 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-17T12:45:11,167 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0d751a622496,42805,1731847510113 from backup master directory
2024-11-17T12:45:11,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x1014a1d67db0001, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:45:11,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0d751a622496,42805,1731847510113
2024-11-17T12:45:11,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:45:11,176 WARN  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T12:45:11,176 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0d751a622496,42805,1731847510113
2024-11-17T12:45:11,179 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-11-17T12:45:11,180 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-11-17T12:45:11,231 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/hbase.id] with ID: 7b5c9bf7-7039-4fc8-af3e-b59bcde6ad8b
2024-11-17T12:45:11,231 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/.tmp/hbase.id
2024-11-17T12:45:11,244 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741826_1002 (size=42)
2024-11-17T12:45:11,244 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741826_1002 (size=42)
2024-11-17T12:45:11,246 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/.tmp/hbase.id]:[hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/hbase.id]
2024-11-17T12:45:11,287 INFO  [master/0d751a622496:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:45:11,292 INFO  [master/0d751a622496:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-17T12:45:11,309 INFO  [master/0d751a622496:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms.
2024-11-17T12:45:11,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x1014a1d67db0001, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:45:11,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:45:11,363 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741827_1003 (size=196)
2024-11-17T12:45:11,363 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741827_1003 (size=196)
2024-11-17T12:45:11,381 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-17T12:45:11,383 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-17T12:45:11,388 INFO  [master/0d751a622496:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T12:45:11,421 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741828_1004 (size=1189)
2024-11-17T12:45:11,421 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741828_1004 (size=1189)
2024-11-17T12:45:11,437 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/data/master/store
2024-11-17T12:45:11,455 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741829_1005 (size=34)
2024-11-17T12:45:11,456 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741829_1005 (size=34)
2024-11-17T12:45:11,460 INFO  [master/0d751a622496:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-11-17T12:45:11,464 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:45:11,465 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T12:45:11,465 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:45:11,466 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:45:11,467 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T12:45:11,467 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:45:11,467 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:45:11,469 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731847511465Disabling compacts and flushes for region at 1731847511465Disabling writes for close at 1731847511467 (+2 ms)Writing region close event to WAL at 1731847511467Closed at 1731847511467
2024-11-17T12:45:11,471 WARN  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/data/master/store/.initializing
2024-11-17T12:45:11,471 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/WALs/0d751a622496,42805,1731847510113
2024-11-17T12:45:11,492 INFO  [master/0d751a622496:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C42805%2C1731847510113, suffix=, logDir=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/WALs/0d751a622496,42805,1731847510113, archiveDir=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/oldWALs, maxLogs=10
2024-11-17T12:45:11,500 INFO  [master/0d751a622496:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C42805%2C1731847510113.1731847511496
2024-11-17T12:45:11,517 INFO  [master/0d751a622496:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/WALs/0d751a622496,42805,1731847510113/0d751a622496%2C42805%2C1731847510113.1731847511496
2024-11-17T12:45:11,525 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38107:38107),(127.0.0.1/127.0.0.1:40369:40369)]
2024-11-17T12:45:11,527 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-17T12:45:11,527 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:45:11,530 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:45:11,531 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:45:11,567 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:45:11,590 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-17T12:45:11,594 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:45:11,596 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:45:11,597 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:45:11,600 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-17T12:45:11,600 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:45:11,601 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:45:11,601 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:45:11,604 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-17T12:45:11,604 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:45:11,605 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:45:11,605 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:45:11,608 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-17T12:45:11,608 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:45:11,609 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:45:11,609 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:45:11,613 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:45:11,615 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:45:11,622 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:45:11,622 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:45:11,625 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-17T12:45:11,629 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:45:11,634 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T12:45:11,635 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=852894, jitterRate=0.0845111757516861}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-17T12:45:11,643 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731847511542Initializing all the Stores at 1731847511544 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847511544Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847511545 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847511545Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847511545Cleaning up temporary data from old regions at 1731847511622 (+77 ms)Region opened successfully at 1731847511643 (+21 ms)
2024-11-17T12:45:11,645 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-17T12:45:11,679 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15bc73a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0d751a622496/172.17.0.2:0
2024-11-17T12:45:11,707 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-17T12:45:11,717 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-17T12:45:11,717 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-17T12:45:11,720 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-17T12:45:11,722 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-17T12:45:11,727 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-17T12:45:11,727 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-17T12:45:11,755 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-17T12:45:11,765 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-17T12:45:11,792 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-17T12:45:11,795 INFO  [master/0d751a622496:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-17T12:45:11,798 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-17T12:45:11,808 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-17T12:45:11,812 INFO  [master/0d751a622496:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-17T12:45:11,818 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-17T12:45:11,825 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-17T12:45:11,827 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-17T12:45:11,833 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-17T12:45:11,849 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-17T12:45:11,858 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-17T12:45:11,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T12:45:11,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x1014a1d67db0001, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T12:45:11,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:45:11,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x1014a1d67db0001, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:45:11,871 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0d751a622496,42805,1731847510113, sessionid=0x1014a1d67db0000, setting cluster-up flag (Was=false)
2024-11-17T12:45:11,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:45:11,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x1014a1d67db0001, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:45:11,925 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-17T12:45:11,930 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0d751a622496,42805,1731847510113
2024-11-17T12:45:11,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:45:11,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x1014a1d67db0001, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:45:11,975 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-17T12:45:11,979 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0d751a622496,42805,1731847510113
2024-11-17T12:45:11,990 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-17T12:45:12,073 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-17T12:45:12,084 INFO  [master/0d751a622496:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-17T12:45:12,088 INFO  [RS:0;0d751a622496:39415 {}] regionserver.HRegionServer(746): ClusterId : 7b5c9bf7-7039-4fc8-af3e-b59bcde6ad8b
2024-11-17T12:45:12,091 DEBUG [RS:0;0d751a622496:39415 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-17T12:45:12,092 INFO  [master/0d751a622496:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-17T12:45:12,098 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0d751a622496,42805,1731847510113
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-17T12:45:12,126 DEBUG [RS:0;0d751a622496:39415 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-17T12:45:12,126 DEBUG [RS:0;0d751a622496:39415 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-17T12:45:12,127 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:45:12,127 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:45:12,127 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:45:12,127 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:45:12,127 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0d751a622496:0, corePoolSize=10, maxPoolSize=10
2024-11-17T12:45:12,127 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:45:12,128 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0d751a622496:0, corePoolSize=2, maxPoolSize=2
2024-11-17T12:45:12,128 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:45:12,134 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731847542134
2024-11-17T12:45:12,135 DEBUG [RS:0;0d751a622496:39415 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-17T12:45:12,135 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T12:45:12,136 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-17T12:45:12,136 DEBUG [RS:0;0d751a622496:39415 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4755102b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0d751a622496/172.17.0.2:0
2024-11-17T12:45:12,136 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-17T12:45:12,137 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-17T12:45:12,141 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:45:12,141 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-17T12:45:12,141 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-17T12:45:12,142 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-17T12:45:12,142 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-17T12:45:12,142 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-17T12:45:12,144 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T12:45:12,150 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-17T12:45:12,151 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-17T12:45:12,152 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-17T12:45:12,154 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-17T12:45:12,154 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-17T12:45:12,155 DEBUG [RS:0;0d751a622496:39415 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0d751a622496:39415
2024-11-17T12:45:12,156 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.large.0-1731847512155,5,FailOnTimeoutGroup]
2024-11-17T12:45:12,157 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.small.0-1731847512156,5,FailOnTimeoutGroup]
2024-11-17T12:45:12,157 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T12:45:12,157 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-17T12:45:12,158 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741831_1007 (size=1321)
2024-11-17T12:45:12,158 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741831_1007 (size=1321)
2024-11-17T12:45:12,158 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-17T12:45:12,159 INFO  [RS:0;0d751a622496:39415 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-17T12:45:12,159 INFO  [RS:0;0d751a622496:39415 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-17T12:45:12,159 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-17T12:45:12,159 DEBUG [RS:0;0d751a622496:39415 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-17T12:45:12,160 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-17T12:45:12,161 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f
2024-11-17T12:45:12,161 INFO  [RS:0;0d751a622496:39415 {}] regionserver.HRegionServer(2659): reportForDuty to master=0d751a622496,42805,1731847510113 with port=39415, startcode=1731847510969
2024-11-17T12:45:12,174 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741832_1008 (size=32)
2024-11-17T12:45:12,174 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741832_1008 (size=32)
2024-11-17T12:45:12,177 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:45:12,179 DEBUG [RS:0;0d751a622496:39415 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-17T12:45:12,180 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T12:45:12,183 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T12:45:12,183 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:45:12,184 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:45:12,184 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T12:45:12,187 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T12:45:12,187 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:45:12,188 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:45:12,188 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T12:45:12,192 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T12:45:12,192 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:45:12,194 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:45:12,194 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T12:45:12,198 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T12:45:12,198 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:45:12,199 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:45:12,199 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T12:45:12,201 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/hbase/meta/1588230740
2024-11-17T12:45:12,202 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/hbase/meta/1588230740
2024-11-17T12:45:12,206 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T12:45:12,206 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T12:45:12,208 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T12:45:12,211 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T12:45:12,216 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T12:45:12,218 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=717121, jitterRate=-0.08813442289829254}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T12:45:12,222 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731847512177Initializing all the Stores at 1731847512179 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847512179Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847512180 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847512180Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847512180Cleaning up temporary data from old regions at 1731847512206 (+26 ms)Region opened successfully at 1731847512222 (+16 ms)
2024-11-17T12:45:12,222 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T12:45:12,222 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T12:45:12,222 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T12:45:12,223 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T12:45:12,223 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T12:45:12,224 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T12:45:12,224 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731847512222Disabling compacts and flushes for region at 1731847512222Disabling writes for close at 1731847512223 (+1 ms)Writing region close event to WAL at 1731847512224 (+1 ms)Closed at 1731847512224
2024-11-17T12:45:12,232 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T12:45:12,232 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-17T12:45:12,239 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-17T12:45:12,250 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T12:45:12,252 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49547, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-11-17T12:45:12,254 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-17T12:45:12,259 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42805 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0d751a622496,39415,1731847510969
2024-11-17T12:45:12,261 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42805 {}] master.ServerManager(517): Registering regionserver=0d751a622496,39415,1731847510969
2024-11-17T12:45:12,279 DEBUG [RS:0;0d751a622496:39415 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f
2024-11-17T12:45:12,279 DEBUG [RS:0;0d751a622496:39415 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38459
2024-11-17T12:45:12,279 DEBUG [RS:0;0d751a622496:39415 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-17T12:45:12,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T12:45:12,320 DEBUG [RS:0;0d751a622496:39415 {}] zookeeper.ZKUtil(111): regionserver:39415-0x1014a1d67db0001, quorum=127.0.0.1:55241, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0d751a622496,39415,1731847510969
2024-11-17T12:45:12,320 WARN  [RS:0;0d751a622496:39415 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T12:45:12,320 INFO  [RS:0;0d751a622496:39415 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T12:45:12,321 DEBUG [RS:0;0d751a622496:39415 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969
2024-11-17T12:45:12,327 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0d751a622496,39415,1731847510969]
2024-11-17T12:45:12,369 INFO  [RS:0;0d751a622496:39415 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-17T12:45:12,390 INFO  [RS:0;0d751a622496:39415 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-17T12:45:12,396 INFO  [RS:0;0d751a622496:39415 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-17T12:45:12,396 INFO  [RS:0;0d751a622496:39415 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:45:12,398 INFO  [RS:0;0d751a622496:39415 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-17T12:45:12,405 INFO  [RS:0;0d751a622496:39415 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-17T12:45:12,405 WARN  [0d751a622496:42805 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-17T12:45:12,407 INFO  [RS:0;0d751a622496:39415 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-17T12:45:12,408 DEBUG [RS:0;0d751a622496:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:45:12,408 DEBUG [RS:0;0d751a622496:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:45:12,408 DEBUG [RS:0;0d751a622496:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:45:12,409 DEBUG [RS:0;0d751a622496:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:45:12,409 DEBUG [RS:0;0d751a622496:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:45:12,409 DEBUG [RS:0;0d751a622496:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0, corePoolSize=2, maxPoolSize=2
2024-11-17T12:45:12,409 DEBUG [RS:0;0d751a622496:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:45:12,409 DEBUG [RS:0;0d751a622496:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:45:12,410 DEBUG [RS:0;0d751a622496:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:45:12,410 DEBUG [RS:0;0d751a622496:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:45:12,410 DEBUG [RS:0;0d751a622496:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:45:12,410 DEBUG [RS:0;0d751a622496:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:45:12,410 DEBUG [RS:0;0d751a622496:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0d751a622496:0, corePoolSize=3, maxPoolSize=3
2024-11-17T12:45:12,411 DEBUG [RS:0;0d751a622496:39415 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0, corePoolSize=3, maxPoolSize=3
2024-11-17T12:45:12,412 INFO  [RS:0;0d751a622496:39415 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T12:45:12,413 INFO  [RS:0;0d751a622496:39415 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T12:45:12,413 INFO  [RS:0;0d751a622496:39415 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:45:12,413 INFO  [RS:0;0d751a622496:39415 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-17T12:45:12,414 INFO  [RS:0;0d751a622496:39415 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-17T12:45:12,414 INFO  [RS:0;0d751a622496:39415 {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,39415,1731847510969-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T12:45:12,439 INFO  [RS:0;0d751a622496:39415 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-17T12:45:12,442 INFO  [RS:0;0d751a622496:39415 {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,39415,1731847510969-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:45:12,442 INFO  [RS:0;0d751a622496:39415 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:45:12,443 INFO  [RS:0;0d751a622496:39415 {}] regionserver.Replication(171): 0d751a622496,39415,1731847510969 started
2024-11-17T12:45:12,467 INFO  [RS:0;0d751a622496:39415 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:45:12,467 INFO  [RS:0;0d751a622496:39415 {}] regionserver.HRegionServer(1482): Serving as 0d751a622496,39415,1731847510969, RpcServer on 0d751a622496/172.17.0.2:39415, sessionid=0x1014a1d67db0001
2024-11-17T12:45:12,468 DEBUG [RS:0;0d751a622496:39415 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-17T12:45:12,468 DEBUG [RS:0;0d751a622496:39415 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0d751a622496,39415,1731847510969
2024-11-17T12:45:12,469 DEBUG [RS:0;0d751a622496:39415 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0d751a622496,39415,1731847510969'
2024-11-17T12:45:12,469 DEBUG [RS:0;0d751a622496:39415 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-17T12:45:12,470 DEBUG [RS:0;0d751a622496:39415 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-17T12:45:12,470 DEBUG [RS:0;0d751a622496:39415 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-17T12:45:12,471 DEBUG [RS:0;0d751a622496:39415 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-17T12:45:12,471 DEBUG [RS:0;0d751a622496:39415 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0d751a622496,39415,1731847510969
2024-11-17T12:45:12,471 DEBUG [RS:0;0d751a622496:39415 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0d751a622496,39415,1731847510969'
2024-11-17T12:45:12,471 DEBUG [RS:0;0d751a622496:39415 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-17T12:45:12,471 DEBUG [RS:0;0d751a622496:39415 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-17T12:45:12,472 DEBUG [RS:0;0d751a622496:39415 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-17T12:45:12,472 INFO  [RS:0;0d751a622496:39415 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-17T12:45:12,473 INFO  [RS:0;0d751a622496:39415 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-17T12:45:12,581 INFO  [RS:0;0d751a622496:39415 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C39415%2C1731847510969, suffix=, logDir=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969, archiveDir=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/oldWALs, maxLogs=32
2024-11-17T12:45:12,585 INFO  [RS:0;0d751a622496:39415 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C39415%2C1731847510969.1731847512585
2024-11-17T12:45:12,605 INFO  [RS:0;0d751a622496:39415 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847512585
2024-11-17T12:45:12,611 DEBUG [RS:0;0d751a622496:39415 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38107:38107),(127.0.0.1/127.0.0.1:40369:40369)]
2024-11-17T12:45:12,658 DEBUG [0d751a622496:42805 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-17T12:45:12,670 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0d751a622496,39415,1731847510969
2024-11-17T12:45:12,677 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0d751a622496,39415,1731847510969, state=OPENING
2024-11-17T12:45:12,715 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-17T12:45:12,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:45:12,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x1014a1d67db0001, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:45:12,726 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:45:12,726 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:45:12,729 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T12:45:12,732 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0d751a622496,39415,1731847510969}]
2024-11-17T12:45:12,914 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-17T12:45:12,918 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58747, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-17T12:45:12,928 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-17T12:45:12,929 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T12:45:12,933 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C39415%2C1731847510969.meta, suffix=.meta, logDir=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969, archiveDir=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/oldWALs, maxLogs=32
2024-11-17T12:45:12,935 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C39415%2C1731847510969.meta.1731847512935.meta
2024-11-17T12:45:12,943 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.meta.1731847512935.meta
2024-11-17T12:45:12,945 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40369:40369),(127.0.0.1/127.0.0.1:38107:38107)]
2024-11-17T12:45:12,946 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-17T12:45:12,948 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-17T12:45:12,950 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-17T12:45:12,954 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-17T12:45:12,958 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-17T12:45:12,958 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:45:12,958 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-17T12:45:12,959 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-17T12:45:12,962 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T12:45:12,963 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T12:45:12,963 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:45:12,964 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:45:12,964 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T12:45:12,966 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T12:45:12,966 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:45:12,967 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:45:12,967 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T12:45:12,968 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T12:45:12,968 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:45:12,969 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:45:12,969 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T12:45:12,970 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T12:45:12,971 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:45:12,971 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:45:12,972 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T12:45:12,973 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/hbase/meta/1588230740
2024-11-17T12:45:12,976 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/hbase/meta/1588230740
2024-11-17T12:45:12,978 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T12:45:12,978 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T12:45:12,979 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T12:45:12,982 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T12:45:12,984 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=870878, jitterRate=0.10737954080104828}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T12:45:12,984 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-17T12:45:12,985 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731847512959Writing region info on filesystem at 1731847512959Initializing all the Stores at 1731847512961 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847512961Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847512961Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847512961Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847512961Cleaning up temporary data from old regions at 1731847512978 (+17 ms)Running coprocessor post-open hooks at 1731847512984 (+6 ms)Region opened successfully at 1731847512985 (+1 ms)
2024-11-17T12:45:12,990 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731847512907
2024-11-17T12:45:13,001 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-17T12:45:13,002 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-17T12:45:13,003 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0d751a622496,39415,1731847510969
2024-11-17T12:45:13,005 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0d751a622496,39415,1731847510969, state=OPEN
2024-11-17T12:45:13,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T12:45:13,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x1014a1d67db0001, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T12:45:13,073 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:45:13,073 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:45:13,073 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0d751a622496,39415,1731847510969
2024-11-17T12:45:13,082 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-17T12:45:13,082 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0d751a622496,39415,1731847510969 in 341 msec
2024-11-17T12:45:13,090 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-17T12:45:13,090 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 845 msec
2024-11-17T12:45:13,092 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T12:45:13,092 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-17T12:45:13,111 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T12:45:13,112 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0d751a622496,39415,1731847510969, seqNum=-1]
2024-11-17T12:45:13,129 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T12:45:13,131 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39565, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T12:45:13,150 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1320 sec
2024-11-17T12:45:13,151 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731847513151, completionTime=-1
2024-11-17T12:45:13,154 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-17T12:45:13,154 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-17T12:45:13,180 INFO  [master/0d751a622496:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-17T12:45:13,180 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731847573180
2024-11-17T12:45:13,181 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731847633181
2024-11-17T12:45:13,181 INFO  [master/0d751a622496:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec
2024-11-17T12:45:13,183 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,42805,1731847510113-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:45:13,183 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,42805,1731847510113-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:45:13,184 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,42805,1731847510113-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:45:13,185 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0d751a622496:42805, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:45:13,185 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-17T12:45:13,186 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-17T12:45:13,191 DEBUG [master/0d751a622496:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-17T12:45:13,212 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.035sec
2024-11-17T12:45:13,213 INFO  [master/0d751a622496:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-17T12:45:13,215 INFO  [master/0d751a622496:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-17T12:45:13,215 INFO  [master/0d751a622496:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-17T12:45:13,216 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-17T12:45:13,216 INFO  [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-17T12:45:13,217 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,42805,1731847510113-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T12:45:13,217 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,42805,1731847510113-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-17T12:45:13,225 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-17T12:45:13,226 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-17T12:45:13,226 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,42805,1731847510113-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:45:13,300 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66cb686a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T12:45:13,302 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-11-17T12:45:13,302 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-11-17T12:45:13,306 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0d751a622496,42805,-1 for getting cluster id
2024-11-17T12:45:13,309 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-17T12:45:13,318 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b5c9bf7-7039-4fc8-af3e-b59bcde6ad8b'
2024-11-17T12:45:13,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-17T12:45:13,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b5c9bf7-7039-4fc8-af3e-b59bcde6ad8b"

2024-11-17T12:45:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c472083, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T12:45:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0d751a622496,42805,-1]
2024-11-17T12:45:13,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-17T12:45:13,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:45:13,331 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-17T12:45:13,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32fb6022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T12:45:13,335 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T12:45:13,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0d751a622496,39415,1731847510969, seqNum=-1]
2024-11-17T12:45:13,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T12:45:13,346 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42236, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T12:45:13,365 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0d751a622496,42805,1731847510113
2024-11-17T12:45:13,365 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:45:13,373 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-17T12:45:13,377 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-17T12:45:13,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 0d751a622496,42805,1731847510113
2024-11-17T12:45:13,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7257518
2024-11-17T12:45:13,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-17T12:45:13,407 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37108, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-17T12:45:13,410 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42805 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-17T12:45:13,410 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42805 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-17T12:45:13,414 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42805 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-17T12:45:13,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42805 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling
2024-11-17T12:45:13,424 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-17T12:45:13,430 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42805 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testSlowSyncLogRolling"
 procId is: 4
2024-11-17T12:45:13,430 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:45:13,433 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-17T12:45:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42805 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-17T12:45:13,471 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741835_1011 (size=389)
2024-11-17T12:45:13,472 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741835_1011 (size=389)
2024-11-17T12:45:13,476 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6f0bd2916164dd42590fc44b55cae5b8, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f
2024-11-17T12:45:13,488 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741836_1012 (size=72)
2024-11-17T12:45:13,488 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741836_1012 (size=72)
2024-11-17T12:45:13,489 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:45:13,490 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 6f0bd2916164dd42590fc44b55cae5b8, disabling compactions & flushes
2024-11-17T12:45:13,490 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8.
2024-11-17T12:45:13,490 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8.
2024-11-17T12:45:13,490 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8. after waiting 0 ms
2024-11-17T12:45:13,490 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8.
2024-11-17T12:45:13,490 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8.
2024-11-17T12:45:13,490 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6f0bd2916164dd42590fc44b55cae5b8:
Waiting for close lock at 1731847513489Disabling compacts and flushes for region at 1731847513489Disabling writes for close at 1731847513490 (+1 ms)Writing region close event to WAL at 1731847513490Closed at 1731847513490
2024-11-17T12:45:13,493 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-17T12:45:13,498 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731847513493"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731847513493"}]},"ts":"1731847513493"}
2024-11-17T12:45:13,503 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-17T12:45:13,505 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-17T12:45:13,508 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731847513505"}]},"ts":"1731847513505"}
2024-11-17T12:45:13,512 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta
2024-11-17T12:45:13,514 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=6f0bd2916164dd42590fc44b55cae5b8, ASSIGN}]
2024-11-17T12:45:13,516 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=6f0bd2916164dd42590fc44b55cae5b8, ASSIGN
2024-11-17T12:45:13,517 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=6f0bd2916164dd42590fc44b55cae5b8, ASSIGN; state=OFFLINE, location=0d751a622496,39415,1731847510969; forceNewPlan=false, retain=false
2024-11-17T12:45:13,672 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6f0bd2916164dd42590fc44b55cae5b8, regionState=OPENING, regionLocation=0d751a622496,39415,1731847510969
2024-11-17T12:45:13,680 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=6f0bd2916164dd42590fc44b55cae5b8, ASSIGN because future has completed
2024-11-17T12:45:13,682 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f0bd2916164dd42590fc44b55cae5b8, server=0d751a622496,39415,1731847510969}]
2024-11-17T12:45:13,844 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8.
2024-11-17T12:45:13,845 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 6f0bd2916164dd42590fc44b55cae5b8, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8.', STARTKEY => '', ENDKEY => ''}
2024-11-17T12:45:13,845 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling 6f0bd2916164dd42590fc44b55cae5b8
2024-11-17T12:45:13,846 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:45:13,846 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 6f0bd2916164dd42590fc44b55cae5b8
2024-11-17T12:45:13,846 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 6f0bd2916164dd42590fc44b55cae5b8
2024-11-17T12:45:13,849 INFO  [StoreOpener-6f0bd2916164dd42590fc44b55cae5b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 6f0bd2916164dd42590fc44b55cae5b8 
2024-11-17T12:45:13,853 INFO  [StoreOpener-6f0bd2916164dd42590fc44b55cae5b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6f0bd2916164dd42590fc44b55cae5b8 columnFamilyName info
2024-11-17T12:45:13,853 DEBUG [StoreOpener-6f0bd2916164dd42590fc44b55cae5b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:45:13,854 INFO  [StoreOpener-6f0bd2916164dd42590fc44b55cae5b8-1 {}] regionserver.HStore(327): Store=6f0bd2916164dd42590fc44b55cae5b8/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:45:13,854 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 6f0bd2916164dd42590fc44b55cae5b8
2024-11-17T12:45:13,856 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8
2024-11-17T12:45:13,856 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8
2024-11-17T12:45:13,857 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 6f0bd2916164dd42590fc44b55cae5b8
2024-11-17T12:45:13,857 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 6f0bd2916164dd42590fc44b55cae5b8
2024-11-17T12:45:13,860 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 6f0bd2916164dd42590fc44b55cae5b8
2024-11-17T12:45:13,863 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T12:45:13,864 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 6f0bd2916164dd42590fc44b55cae5b8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=787341, jitterRate=0.0011567175388336182}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-17T12:45:13,864 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6f0bd2916164dd42590fc44b55cae5b8
2024-11-17T12:45:13,865 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 6f0bd2916164dd42590fc44b55cae5b8:
Running coprocessor pre-open hook at 1731847513846Writing region info on filesystem at 1731847513846Initializing all the Stores at 1731847513849 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847513849Cleaning up temporary data from old regions at 1731847513857 (+8 ms)Running coprocessor post-open hooks at 1731847513864 (+7 ms)Region opened successfully at 1731847513865 (+1 ms)
2024-11-17T12:45:13,867 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8., pid=6, masterSystemTime=1731847513837
2024-11-17T12:45:13,871 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8.
2024-11-17T12:45:13,871 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8.
2024-11-17T12:45:13,872 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6f0bd2916164dd42590fc44b55cae5b8, regionState=OPEN, openSeqNum=2, regionLocation=0d751a622496,39415,1731847510969
2024-11-17T12:45:13,876 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f0bd2916164dd42590fc44b55cae5b8, server=0d751a622496,39415,1731847510969 because future has completed
2024-11-17T12:45:13,883 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-17T12:45:13,883 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6f0bd2916164dd42590fc44b55cae5b8, server=0d751a622496,39415,1731847510969 in 197 msec
2024-11-17T12:45:13,887 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-17T12:45:13,887 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=6f0bd2916164dd42590fc44b55cae5b8, ASSIGN in 370 msec
2024-11-17T12:45:13,889 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-17T12:45:13,890 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731847513889"}]},"ts":"1731847513889"}
2024-11-17T12:45:13,893 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta
2024-11-17T12:45:13,895 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-17T12:45:13,899 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 479 msec
2024-11-17T12:45:18,472 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-17T12:45:18,522 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-17T12:45:18,523 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling'
2024-11-17T12:45:20,480 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-17T12:45:20,481 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-17T12:45:20,483 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-17T12:45:20,483 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-17T12:45:20,485 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T12:45:20,485 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-17T12:45:20,485 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-17T12:45:20,485 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-17T12:45:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42805 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-17T12:45:23,491 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed
2024-11-17T12:45:23,495 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100
2024-11-17T12:45:23,502 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling
2024-11-17T12:45:23,503 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8.
2024-11-17T12:45:23,504 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C39415%2C1731847510969.1731847523504
2024-11-17T12:45:23,517 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:45:23,517 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:45:23,518 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:45:23,519 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:45:23,519 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:45:23,519 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847512585 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847523504
2024-11-17T12:45:23,525 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741833_1009 (size=451)
2024-11-17T12:45:23,526 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741833_1009 (size=451)
2024-11-17T12:45:23,533 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847512585 to hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/oldWALs/0d751a622496%2C39415%2C1731847510969.1731847512585
2024-11-17T12:45:23,539 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40369:40369),(127.0.0.1/127.0.0.1:38107:38107)]
2024-11-17T12:45:23,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8., hostname=0d751a622496,39415,1731847510969, seqNum=2]
2024-11-17T12:45:35,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39415 {}] regionserver.HRegion(8855): Flush requested on 6f0bd2916164dd42590fc44b55cae5b8
2024-11-17T12:45:35,589 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f0bd2916164dd42590fc44b55cae5b8 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-17T12:45:35,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/.tmp/info/031a613de5f54fc59328da88cc36d4ba is 1080, key is row0001/info:/1731847523552/Put/seqid=0
2024-11-17T12:45:35,690 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741838_1014 (size=12509)
2024-11-17T12:45:35,690 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741838_1014 (size=12509)
2024-11-17T12:45:35,692 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/.tmp/info/031a613de5f54fc59328da88cc36d4ba
2024-11-17T12:45:35,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/.tmp/info/031a613de5f54fc59328da88cc36d4ba as hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/031a613de5f54fc59328da88cc36d4ba
2024-11-17T12:45:35,784 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/031a613de5f54fc59328da88cc36d4ba, entries=7, sequenceid=11, filesize=12.2 K
2024-11-17T12:45:35,791 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 6f0bd2916164dd42590fc44b55cae5b8 in 203ms, sequenceid=11, compaction requested=false
2024-11-17T12:45:35,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f0bd2916164dd42590fc44b55cae5b8:

2024-11-17T12:45:39,221 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-17T12:45:43,604 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C39415%2C1731847510969.1731847543604
2024-11-17T12:45:43,815 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 208 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43907,DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf,DISK], DatanodeInfoWithStorage[127.0.0.1:36729,DS-0952eab7-880d-498a-bdfe-f601e6454cfd,DISK]]
2024-11-17T12:45:43,816 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:45:43,816 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:45:43,816 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:45:43,816 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:45:43,816 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:45:43,816 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847523504 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847543604
2024-11-17T12:45:43,819 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741837_1013 (size=12399)
2024-11-17T12:45:43,819 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741837_1013 (size=12399)
2024-11-17T12:45:43,825 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38107:38107),(127.0.0.1/127.0.0.1:40369:40369)]
2024-11-17T12:45:44,029 INFO  [FSHLog-0-hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f-prefix:0d751a622496,39415,1731847510969 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36729,DS-0952eab7-880d-498a-bdfe-f601e6454cfd,DISK], DatanodeInfoWithStorage[127.0.0.1:43907,DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf,DISK]]
2024-11-17T12:45:46,233 INFO  [FSHLog-0-hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f-prefix:0d751a622496,39415,1731847510969 {}] wal.AbstractFSWAL(1368): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36729,DS-0952eab7-880d-498a-bdfe-f601e6454cfd,DISK], DatanodeInfoWithStorage[127.0.0.1:43907,DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf,DISK]]
2024-11-17T12:45:48,440 INFO  [FSHLog-0-hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f-prefix:0d751a622496,39415,1731847510969 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36729,DS-0952eab7-880d-498a-bdfe-f601e6454cfd,DISK], DatanodeInfoWithStorage[127.0.0.1:43907,DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf,DISK]]
2024-11-17T12:45:50,645 INFO  [FSHLog-0-hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f-prefix:0d751a622496,39415,1731847510969 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36729,DS-0952eab7-880d-498a-bdfe-f601e6454cfd,DISK], DatanodeInfoWithStorage[127.0.0.1:43907,DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf,DISK]]
2024-11-17T12:45:50,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39415 {}] regionserver.HRegion(8855): Flush requested on 6f0bd2916164dd42590fc44b55cae5b8
2024-11-17T12:45:50,646 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f0bd2916164dd42590fc44b55cae5b8 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-17T12:45:50,850 INFO  [FSHLog-0-hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f-prefix:0d751a622496,39415,1731847510969 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36729,DS-0952eab7-880d-498a-bdfe-f601e6454cfd,DISK], DatanodeInfoWithStorage[127.0.0.1:43907,DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf,DISK]]
2024-11-17T12:45:50,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/.tmp/info/34e95fc47230423ea9dbb6956e958058 is 1080, key is row0008/info:/1731847537588/Put/seqid=0
2024-11-17T12:45:50,868 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741840_1016 (size=12509)
2024-11-17T12:45:50,868 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741840_1016 (size=12509)
2024-11-17T12:45:50,869 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/.tmp/info/34e95fc47230423ea9dbb6956e958058
2024-11-17T12:45:50,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/.tmp/info/34e95fc47230423ea9dbb6956e958058 as hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/34e95fc47230423ea9dbb6956e958058
2024-11-17T12:45:50,892 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/34e95fc47230423ea9dbb6956e958058, entries=7, sequenceid=21, filesize=12.2 K
2024-11-17T12:45:51,095 INFO  [FSHLog-0-hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f-prefix:0d751a622496,39415,1731847510969 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36729,DS-0952eab7-880d-498a-bdfe-f601e6454cfd,DISK], DatanodeInfoWithStorage[127.0.0.1:43907,DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf,DISK]]
2024-11-17T12:45:51,095 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 6f0bd2916164dd42590fc44b55cae5b8 in 448ms, sequenceid=21, compaction requested=false
2024-11-17T12:45:51,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f0bd2916164dd42590fc44b55cae5b8:

2024-11-17T12:45:51,096 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K
2024-11-17T12:45:51,096 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:45:51,097 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/031a613de5f54fc59328da88cc36d4ba because midkey is the same as first or last row
2024-11-17T12:45:52,851 INFO  [FSHLog-0-hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f-prefix:0d751a622496,39415,1731847510969 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36729,DS-0952eab7-880d-498a-bdfe-f601e6454cfd,DISK], DatanodeInfoWithStorage[127.0.0.1:43907,DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf,DISK]]
2024-11-17T12:45:53,227 INFO  [master/0d751a622496:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-17T12:45:53,227 INFO  [master/0d751a622496:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-17T12:45:55,057 INFO  [FSHLog-0-hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f-prefix:0d751a622496,39415,1731847510969 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36729,DS-0952eab7-880d-498a-bdfe-f601e6454cfd,DISK], DatanodeInfoWithStorage[127.0.0.1:43907,DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf,DISK]]
2024-11-17T12:45:55,060 WARN  [FSHLog-0-hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f-prefix:0d751a622496,39415,1731847510969 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36729,DS-0952eab7-880d-498a-bdfe-f601e6454cfd,DISK], DatanodeInfoWithStorage[127.0.0.1:43907,DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf,DISK]]
2024-11-17T12:45:55,061 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0d751a622496%2C39415%2C1731847510969:(num 1731847543604) roll requested
2024-11-17T12:45:55,061 INFO  [regionserver/0d751a622496:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C39415%2C1731847510969.1731847555061
2024-11-17T12:45:55,274 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 210 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36729,DS-0952eab7-880d-498a-bdfe-f601e6454cfd,DISK], DatanodeInfoWithStorage[127.0.0.1:43907,DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf,DISK]]
2024-11-17T12:45:55,274 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:45:55,274 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:45:55,275 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:45:55,275 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:45:55,275 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:45:55,275 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847543604 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847555061
2024-11-17T12:45:55,278 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741839_1015 (size=7739)
2024-11-17T12:45:55,278 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741839_1015 (size=7739)
2024-11-17T12:45:55,283 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40369:40369),(127.0.0.1/127.0.0.1:38107:38107)]
2024-11-17T12:45:55,283 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847543604 is not closed yet, will try archiving it next time
2024-11-17T12:45:55,283 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847523504 to hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/oldWALs/0d751a622496%2C39415%2C1731847510969.1731847523504
2024-11-17T12:45:57,262 INFO  [FSHLog-0-hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f-prefix:0d751a622496,39415,1731847510969 {}] wal.AbstractFSWAL(1368): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43907,DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf,DISK], DatanodeInfoWithStorage[127.0.0.1:36729,DS-0952eab7-880d-498a-bdfe-f601e6454cfd,DISK]]
2024-11-17T12:45:58,846 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 6f0bd2916164dd42590fc44b55cae5b8, had cached 0 bytes from a total of 25018
2024-11-17T12:45:59,469 INFO  [FSHLog-0-hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f-prefix:0d751a622496,39415,1731847510969 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43907,DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf,DISK], DatanodeInfoWithStorage[127.0.0.1:36729,DS-0952eab7-880d-498a-bdfe-f601e6454cfd,DISK]]
2024-11-17T12:46:01,676 INFO  [FSHLog-0-hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f-prefix:0d751a622496,39415,1731847510969 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43907,DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf,DISK], DatanodeInfoWithStorage[127.0.0.1:36729,DS-0952eab7-880d-498a-bdfe-f601e6454cfd,DISK]]
2024-11-17T12:46:03,884 INFO  [FSHLog-0-hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f-prefix:0d751a622496,39415,1731847510969 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43907,DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf,DISK], DatanodeInfoWithStorage[127.0.0.1:36729,DS-0952eab7-880d-498a-bdfe-f601e6454cfd,DISK]]
2024-11-17T12:46:05,888 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-17T12:46:05,890 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C39415%2C1731847510969.1731847565889
2024-11-17T12:46:09,221 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-17T12:46:10,903 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5007 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43907,DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf,DISK], DatanodeInfoWithStorage[127.0.0.1:36729,DS-0952eab7-880d-498a-bdfe-f601e6454cfd,DISK]]
2024-11-17T12:46:10,906 WARN  [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5007 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43907,DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf,DISK], DatanodeInfoWithStorage[127.0.0.1:36729,DS-0952eab7-880d-498a-bdfe-f601e6454cfd,DISK]]
2024-11-17T12:46:10,906 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0d751a622496%2C39415%2C1731847510969:(num 1731847565889) roll requested
2024-11-17T12:46:10,906 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:10,906 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:10,907 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:10,907 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:10,907 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:10,907 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847555061 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847565889
2024-11-17T12:46:10,909 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741841_1017 (size=4753)
2024-11-17T12:46:10,910 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741841_1017 (size=4753)
2024-11-17T12:46:10,922 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38107:38107),(127.0.0.1/127.0.0.1:40369:40369)]
2024-11-17T12:46:10,922 INFO  [regionserver/0d751a622496:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C39415%2C1731847510969.1731847570922
2024-11-17T12:46:15,925 INFO  [FSHLog-0-hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f-prefix:0d751a622496,39415,1731847510969 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36729,DS-0952eab7-880d-498a-bdfe-f601e6454cfd,DISK], DatanodeInfoWithStorage[127.0.0.1:43907,DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf,DISK]]
2024-11-17T12:46:15,925 WARN  [FSHLog-0-hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f-prefix:0d751a622496,39415,1731847510969 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36729,DS-0952eab7-880d-498a-bdfe-f601e6454cfd,DISK], DatanodeInfoWithStorage[127.0.0.1:43907,DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf,DISK]]
2024-11-17T12:46:15,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39415 {}] regionserver.HRegion(8855): Flush requested on 6f0bd2916164dd42590fc44b55cae5b8
2024-11-17T12:46:15,926 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f0bd2916164dd42590fc44b55cae5b8 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-17T12:46:15,932 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5007 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36729,DS-0952eab7-880d-498a-bdfe-f601e6454cfd,DISK], DatanodeInfoWithStorage[127.0.0.1:43907,DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf,DISK]]
2024-11-17T12:46:15,932 WARN  [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5007 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36729,DS-0952eab7-880d-498a-bdfe-f601e6454cfd,DISK], DatanodeInfoWithStorage[127.0.0.1:43907,DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf,DISK]]
2024-11-17T12:46:17,926 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-17T12:46:20,928 INFO  [FSHLog-0-hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f-prefix:0d751a622496,39415,1731847510969 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36729,DS-0952eab7-880d-498a-bdfe-f601e6454cfd,DISK], DatanodeInfoWithStorage[127.0.0.1:43907,DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf,DISK]]
2024-11-17T12:46:20,928 WARN  [FSHLog-0-hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f-prefix:0d751a622496,39415,1731847510969 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36729,DS-0952eab7-880d-498a-bdfe-f601e6454cfd,DISK], DatanodeInfoWithStorage[127.0.0.1:43907,DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf,DISK]]
2024-11-17T12:46:20,928 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:20,928 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:20,929 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:20,929 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:20,929 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:20,929 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847565889 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847570922
2024-11-17T12:46:20,930 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40369:40369),(127.0.0.1/127.0.0.1:38107:38107)]
2024-11-17T12:46:20,931 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847565889 is not closed yet, will try archiving it next time
2024-11-17T12:46:20,931 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0d751a622496%2C39415%2C1731847510969:(num 1731847570922) roll requested
2024-11-17T12:46:20,931 INFO  [regionserver/0d751a622496:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C39415%2C1731847510969.1731847580931
2024-11-17T12:46:20,932 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741842_1018 (size=1569)
2024-11-17T12:46:20,932 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741842_1018 (size=1569)
2024-11-17T12:46:20,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/.tmp/info/492ab984b65e4db0bcc0c015388e7fbc is 1080, key is row0015/info:/1731847552649/Put/seqid=0
2024-11-17T12:46:20,945 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741844_1020 (size=12509)
2024-11-17T12:46:20,946 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741844_1020 (size=12509)
2024-11-17T12:46:20,946 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/.tmp/info/492ab984b65e4db0bcc0c015388e7fbc
2024-11-17T12:46:20,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/.tmp/info/492ab984b65e4db0bcc0c015388e7fbc as hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/492ab984b65e4db0bcc0c015388e7fbc
2024-11-17T12:46:20,970 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/492ab984b65e4db0bcc0c015388e7fbc, entries=7, sequenceid=31, filesize=12.2 K
2024-11-17T12:46:25,943 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5006 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43907,DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf,DISK], DatanodeInfoWithStorage[127.0.0.1:36729,DS-0952eab7-880d-498a-bdfe-f601e6454cfd,DISK]]
2024-11-17T12:46:25,943 WARN  [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5006 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43907,DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf,DISK], DatanodeInfoWithStorage[127.0.0.1:36729,DS-0952eab7-880d-498a-bdfe-f601e6454cfd,DISK]]
2024-11-17T12:46:25,972 INFO  [FSHLog-0-hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f-prefix:0d751a622496,39415,1731847510969 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43907,DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf,DISK], DatanodeInfoWithStorage[127.0.0.1:36729,DS-0952eab7-880d-498a-bdfe-f601e6454cfd,DISK]]
2024-11-17T12:46:25,972 WARN  [FSHLog-0-hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f-prefix:0d751a622496,39415,1731847510969 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43907,DS-0c6bbff4-eef7-4fed-80c0-80c834bf1caf,DISK], DatanodeInfoWithStorage[127.0.0.1:36729,DS-0952eab7-880d-498a-bdfe-f601e6454cfd,DISK]]
2024-11-17T12:46:25,972 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 6f0bd2916164dd42590fc44b55cae5b8 in 10047ms, sequenceid=31, compaction requested=true
2024-11-17T12:46:25,972 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:25,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f0bd2916164dd42590fc44b55cae5b8:

2024-11-17T12:46:25,973 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:25,973 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:25,973 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K
2024-11-17T12:46:25,973 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:46:25,973 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:25,973 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/031a613de5f54fc59328da88cc36d4ba because midkey is the same as first or last row
2024-11-17T12:46:25,973 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:25,973 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847570922 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847580931
2024-11-17T12:46:25,974 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38107:38107),(127.0.0.1/127.0.0.1:40369:40369)]
2024-11-17T12:46:25,974 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847570922 is not closed yet, will try archiving it next time
2024-11-17T12:46:25,974 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0d751a622496%2C39415%2C1731847510969:(num 1731847580931) roll requested
2024-11-17T12:46:25,974 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847543604 to hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/oldWALs/0d751a622496%2C39415%2C1731847510969.1731847543604
2024-11-17T12:46:25,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f0bd2916164dd42590fc44b55cae5b8:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T12:46:25,975 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C39415%2C1731847510969.1731847585974
2024-11-17T12:46:25,977 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741843_1019 (size=438)
2024-11-17T12:46:25,977 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847555061 to hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/oldWALs/0d751a622496%2C39415%2C1731847510969.1731847555061
2024-11-17T12:46:25,977 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741843_1019 (size=438)
2024-11-17T12:46:25,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:46:25,978 DEBUG [RS:0;0d751a622496:39415-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T12:46:25,979 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847565889 to hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/oldWALs/0d751a622496%2C39415%2C1731847510969.1731847565889
2024-11-17T12:46:25,982 DEBUG [RS:0;0d751a622496:39415-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-17T12:46:25,984 DEBUG [RS:0;0d751a622496:39415-shortCompactions-0 {}] regionserver.HStore(1541): 6f0bd2916164dd42590fc44b55cae5b8/info is initiating minor compaction (all files)
2024-11-17T12:46:25,984 INFO  [RS:0;0d751a622496:39415-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f0bd2916164dd42590fc44b55cae5b8/info in TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8.
2024-11-17T12:46:25,985 INFO  [RS:0;0d751a622496:39415-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/031a613de5f54fc59328da88cc36d4ba, hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/34e95fc47230423ea9dbb6956e958058, hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/492ab984b65e4db0bcc0c015388e7fbc] into tmpdir=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/.tmp, totalSize=36.6 K
2024-11-17T12:46:25,986 DEBUG [RS:0;0d751a622496:39415-shortCompactions-0 {}] compactions.Compactor(225): Compacting 031a613de5f54fc59328da88cc36d4ba, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731847523552
2024-11-17T12:46:25,987 DEBUG [RS:0;0d751a622496:39415-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34e95fc47230423ea9dbb6956e958058, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1731847537588
2024-11-17T12:46:25,988 DEBUG [RS:0;0d751a622496:39415-shortCompactions-0 {}] compactions.Compactor(225): Compacting 492ab984b65e4db0bcc0c015388e7fbc, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1731847552649
2024-11-17T12:46:25,988 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:25,988 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:25,988 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:25,989 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:25,989 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:25,989 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847580931 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847585974
2024-11-17T12:46:25,991 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741845_1021 (size=93)
2024-11-17T12:46:25,991 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741845_1021 (size=93)
2024-11-17T12:46:25,992 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847570922 is not closed yet, will try archiving it next time
2024-11-17T12:46:25,992 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847580931 to hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/oldWALs/0d751a622496%2C39415%2C1731847510969.1731847580931
2024-11-17T12:46:25,999 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40369:40369),(127.0.0.1/127.0.0.1:38107:38107)]
2024-11-17T12:46:25,999 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847570922 is not closed yet, will try archiving it next time
2024-11-17T12:46:25,999 INFO  [regionserver/0d751a622496:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C39415%2C1731847510969.1731847585999
2024-11-17T12:46:26,008 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:26,008 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:26,008 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:26,008 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:26,009 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:26,009 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847585974 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847585999
2024-11-17T12:46:26,012 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40369:40369),(127.0.0.1/127.0.0.1:38107:38107)]
2024-11-17T12:46:26,012 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847570922 is not closed yet, will try archiving it next time
2024-11-17T12:46:26,012 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847585974 is not closed yet, will try archiving it next time
2024-11-17T12:46:26,013 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741846_1022 (size=1258)
2024-11-17T12:46:26,013 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741846_1022 (size=1258)
2024-11-17T12:46:26,014 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847570922 is not closed yet, will try archiving it next time
2024-11-17T12:46:26,021 INFO  [RS:0;0d751a622496:39415-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f0bd2916164dd42590fc44b55cae5b8#info#compaction#3 average throughput is 10.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T12:46:26,022 DEBUG [RS:0;0d751a622496:39415-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/.tmp/info/3524f70c52ec48158377c80011414195 is 1080, key is row0001/info:/1731847523552/Put/seqid=0
2024-11-17T12:46:26,033 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741848_1024 (size=27710)
2024-11-17T12:46:26,033 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741848_1024 (size=27710)
2024-11-17T12:46:26,046 DEBUG [RS:0;0d751a622496:39415-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/.tmp/info/3524f70c52ec48158377c80011414195 as hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/3524f70c52ec48158377c80011414195
2024-11-17T12:46:26,065 INFO  [RS:0;0d751a622496:39415-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f0bd2916164dd42590fc44b55cae5b8/info of 6f0bd2916164dd42590fc44b55cae5b8 into 3524f70c52ec48158377c80011414195(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T12:46:26,065 DEBUG [RS:0;0d751a622496:39415-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f0bd2916164dd42590fc44b55cae5b8:

2024-11-17T12:46:26,067 INFO  [RS:0;0d751a622496:39415-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8., storeName=6f0bd2916164dd42590fc44b55cae5b8/info, priority=13, startTime=1731847585974; duration=0sec
2024-11-17T12:46:26,067 DEBUG [RS:0;0d751a622496:39415-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-17T12:46:26,067 DEBUG [RS:0;0d751a622496:39415-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:46:26,067 DEBUG [RS:0;0d751a622496:39415-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/3524f70c52ec48158377c80011414195 because midkey is the same as first or last row
2024-11-17T12:46:26,068 DEBUG [RS:0;0d751a622496:39415-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-17T12:46:26,068 DEBUG [RS:0;0d751a622496:39415-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:46:26,068 DEBUG [RS:0;0d751a622496:39415-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/3524f70c52ec48158377c80011414195 because midkey is the same as first or last row
2024-11-17T12:46:26,068 DEBUG [RS:0;0d751a622496:39415-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-17T12:46:26,068 DEBUG [RS:0;0d751a622496:39415-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:46:26,068 DEBUG [RS:0;0d751a622496:39415-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/3524f70c52ec48158377c80011414195 because midkey is the same as first or last row
2024-11-17T12:46:26,068 DEBUG [RS:0;0d751a622496:39415-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:46:26,069 DEBUG [RS:0;0d751a622496:39415-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f0bd2916164dd42590fc44b55cae5b8:info
2024-11-17T12:46:26,379 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/WALs/0d751a622496,39415,1731847510969/0d751a622496%2C39415%2C1731847510969.1731847570922 to hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/oldWALs/0d751a622496%2C39415%2C1731847510969.1731847570922
2024-11-17T12:46:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39415 {}] regionserver.HRegion(8855): Flush requested on 6f0bd2916164dd42590fc44b55cae5b8
2024-11-17T12:46:38,029 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f0bd2916164dd42590fc44b55cae5b8 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-17T12:46:38,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/.tmp/info/738c7ff36b2b49beaec2fa5cf00a10ce is 1080, key is row0022/info:/1731847586001/Put/seqid=0
2024-11-17T12:46:38,042 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741849_1025 (size=12509)
2024-11-17T12:46:38,043 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741849_1025 (size=12509)
2024-11-17T12:46:38,043 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/.tmp/info/738c7ff36b2b49beaec2fa5cf00a10ce
2024-11-17T12:46:38,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/.tmp/info/738c7ff36b2b49beaec2fa5cf00a10ce as hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/738c7ff36b2b49beaec2fa5cf00a10ce
2024-11-17T12:46:38,066 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/738c7ff36b2b49beaec2fa5cf00a10ce, entries=7, sequenceid=42, filesize=12.2 K
2024-11-17T12:46:38,068 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 6f0bd2916164dd42590fc44b55cae5b8 in 40ms, sequenceid=42, compaction requested=false
2024-11-17T12:46:38,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f0bd2916164dd42590fc44b55cae5b8:

2024-11-17T12:46:38,068 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K
2024-11-17T12:46:38,068 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:46:38,069 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/3524f70c52ec48158377c80011414195 because midkey is the same as first or last row
2024-11-17T12:46:39,221 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-17T12:46:43,846 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 6f0bd2916164dd42590fc44b55cae5b8, had cached 0 bytes from a total of 40219
2024-11-17T12:46:46,042 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-17T12:46:46,042 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T12:46:46,043 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T12:46:46,050 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:46:46,051 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:46:46,051 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-17T12:46:46,051 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-17T12:46:46,051 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=523817283, stopped=false
2024-11-17T12:46:46,052 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0d751a622496,42805,1731847510113
2024-11-17T12:46:46,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T12:46:46,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x1014a1d67db0001, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T12:46:46,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:46,104 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T12:46:46,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x1014a1d67db0001, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:46,104 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T12:46:46,105 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T12:46:46,105 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:46:46,105 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39415-0x1014a1d67db0001, quorum=127.0.0.1:55241, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:46:46,105 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:46:46,105 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0d751a622496,39415,1731847510969' *****
2024-11-17T12:46:46,105 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-17T12:46:46,106 INFO  [RS:0;0d751a622496:39415 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-17T12:46:46,106 INFO  [RS:0;0d751a622496:39415 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-17T12:46:46,106 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-17T12:46:46,106 INFO  [RS:0;0d751a622496:39415 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-17T12:46:46,107 INFO  [RS:0;0d751a622496:39415 {}] regionserver.HRegionServer(3091): Received CLOSE for 6f0bd2916164dd42590fc44b55cae5b8
2024-11-17T12:46:46,107 INFO  [RS:0;0d751a622496:39415 {}] regionserver.HRegionServer(959): stopping server 0d751a622496,39415,1731847510969
2024-11-17T12:46:46,107 INFO  [RS:0;0d751a622496:39415 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T12:46:46,107 INFO  [RS:0;0d751a622496:39415 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0d751a622496:39415.
2024-11-17T12:46:46,108 DEBUG [RS:0;0d751a622496:39415 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T12:46:46,108 DEBUG [RS:0;0d751a622496:39415 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:46:46,108 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6f0bd2916164dd42590fc44b55cae5b8, disabling compactions & flushes
2024-11-17T12:46:46,108 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8.
2024-11-17T12:46:46,108 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8.
2024-11-17T12:46:46,108 INFO  [RS:0;0d751a622496:39415 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-17T12:46:46,108 INFO  [RS:0;0d751a622496:39415 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-17T12:46:46,108 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8. after waiting 0 ms
2024-11-17T12:46:46,108 INFO  [RS:0;0d751a622496:39415 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-17T12:46:46,108 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8.
2024-11-17T12:46:46,108 INFO  [RS:0;0d751a622496:39415 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-17T12:46:46,108 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 6f0bd2916164dd42590fc44b55cae5b8 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-11-17T12:46:46,109 INFO  [RS:0;0d751a622496:39415 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-17T12:46:46,109 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T12:46:46,109 DEBUG [RS:0;0d751a622496:39415 {}] regionserver.HRegionServer(1325): Online Regions={6f0bd2916164dd42590fc44b55cae5b8=TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8., 1588230740=hbase:meta,,1.1588230740}
2024-11-17T12:46:46,109 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T12:46:46,109 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T12:46:46,109 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T12:46:46,109 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T12:46:46,109 DEBUG [RS:0;0d751a622496:39415 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 6f0bd2916164dd42590fc44b55cae5b8
2024-11-17T12:46:46,110 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB
2024-11-17T12:46:46,115 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/.tmp/info/b03b0943d3ee40e89ec2d0d6aa2be0df is 1080, key is row0029/info:/1731847600031/Put/seqid=0
2024-11-17T12:46:46,123 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741850_1026 (size=8193)
2024-11-17T12:46:46,123 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741850_1026 (size=8193)
2024-11-17T12:46:46,124 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/.tmp/info/b03b0943d3ee40e89ec2d0d6aa2be0df
2024-11-17T12:46:46,133 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/hbase/meta/1588230740/.tmp/info/c364bb49e6b6436b944ba9f8cf53693f is 195, key is TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8./info:regioninfo/1731847513872/Put/seqid=0
2024-11-17T12:46:46,134 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/.tmp/info/b03b0943d3ee40e89ec2d0d6aa2be0df as hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/b03b0943d3ee40e89ec2d0d6aa2be0df
2024-11-17T12:46:46,139 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741851_1027 (size=7016)
2024-11-17T12:46:46,140 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741851_1027 (size=7016)
2024-11-17T12:46:46,140 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/hbase/meta/1588230740/.tmp/info/c364bb49e6b6436b944ba9f8cf53693f
2024-11-17T12:46:46,143 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/b03b0943d3ee40e89ec2d0d6aa2be0df, entries=3, sequenceid=48, filesize=8.0 K
2024-11-17T12:46:46,145 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 6f0bd2916164dd42590fc44b55cae5b8 in 37ms, sequenceid=48, compaction requested=true
2024-11-17T12:46:46,145 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/031a613de5f54fc59328da88cc36d4ba, hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/34e95fc47230423ea9dbb6956e958058, hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/492ab984b65e4db0bcc0c015388e7fbc] to archive
2024-11-17T12:46:46,148 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-17T12:46:46,152 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/031a613de5f54fc59328da88cc36d4ba to hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/archive/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/031a613de5f54fc59328da88cc36d4ba
2024-11-17T12:46:46,155 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/34e95fc47230423ea9dbb6956e958058 to hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/archive/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/34e95fc47230423ea9dbb6956e958058
2024-11-17T12:46:46,156 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/492ab984b65e4db0bcc0c015388e7fbc to hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/archive/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/info/492ab984b65e4db0bcc0c015388e7fbc
2024-11-17T12:46:46,169 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/hbase/meta/1588230740/.tmp/ns/704e4799ad624afebeb5a0306831c86e is 43, key is default/ns:d/1731847513135/Put/seqid=0
2024-11-17T12:46:46,170 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=0d751a622496:42805 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-11-17T12:46:46,176 WARN  [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [031a613de5f54fc59328da88cc36d4ba=12509, 34e95fc47230423ea9dbb6956e958058=12509, 492ab984b65e4db0bcc0c015388e7fbc=12509]
2024-11-17T12:46:46,179 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741852_1028 (size=5153)
2024-11-17T12:46:46,179 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741852_1028 (size=5153)
2024-11-17T12:46:46,188 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/hbase/meta/1588230740/.tmp/ns/704e4799ad624afebeb5a0306831c86e
2024-11-17T12:46:46,199 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/default/TestLogRolling-testSlowSyncLogRolling/6f0bd2916164dd42590fc44b55cae5b8/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1
2024-11-17T12:46:46,202 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8.
2024-11-17T12:46:46,202 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6f0bd2916164dd42590fc44b55cae5b8:
Waiting for close lock at 1731847606107Running coprocessor pre-close hooks at 1731847606108 (+1 ms)Disabling compacts and flushes for region at 1731847606108Disabling writes for close at 1731847606108Obtaining lock to block concurrent updates at 1731847606108Preparing flush snapshotting stores in 6f0bd2916164dd42590fc44b55cae5b8 at 1731847606108Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1731847606109 (+1 ms)Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8. at 1731847606110 (+1 ms)Flushing 6f0bd2916164dd42590fc44b55cae5b8/info: creating writer at 1731847606110Flushing 6f0bd2916164dd42590fc44b55cae5b8/info: appending metadata at 1731847606114 (+4 ms)Flushing 6f0bd2916164dd42590fc44b55cae5b8/info: closing flushed file at 1731847606114Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ad736a5: reopening flushed file at 1731847606132 (+18 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 6f0bd2916164dd42590fc44b55cae5b8 in 37ms, sequenceid=48, compaction requested=true at 1731847606145 (+13 ms)Writing region close event to WAL at 1731847606182 (+37 ms)Running coprocessor post-close hooks at 1731847606200 (+18 ms)Closed at 1731847606202 (+2 ms)
2024-11-17T12:46:46,203 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1731847513409.6f0bd2916164dd42590fc44b55cae5b8.
2024-11-17T12:46:46,214 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/hbase/meta/1588230740/.tmp/table/87d9eb8168774d27a18dd25f4d13d175 is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1731847513889/Put/seqid=0
2024-11-17T12:46:46,224 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741853_1029 (size=5396)
2024-11-17T12:46:46,224 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741853_1029 (size=5396)
2024-11-17T12:46:46,225 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/hbase/meta/1588230740/.tmp/table/87d9eb8168774d27a18dd25f4d13d175
2024-11-17T12:46:46,234 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/hbase/meta/1588230740/.tmp/info/c364bb49e6b6436b944ba9f8cf53693f as hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/hbase/meta/1588230740/info/c364bb49e6b6436b944ba9f8cf53693f
2024-11-17T12:46:46,248 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/hbase/meta/1588230740/info/c364bb49e6b6436b944ba9f8cf53693f, entries=10, sequenceid=11, filesize=6.9 K
2024-11-17T12:46:46,249 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/hbase/meta/1588230740/.tmp/ns/704e4799ad624afebeb5a0306831c86e as hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/hbase/meta/1588230740/ns/704e4799ad624afebeb5a0306831c86e
2024-11-17T12:46:46,269 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/hbase/meta/1588230740/ns/704e4799ad624afebeb5a0306831c86e, entries=2, sequenceid=11, filesize=5.0 K
2024-11-17T12:46:46,271 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/hbase/meta/1588230740/.tmp/table/87d9eb8168774d27a18dd25f4d13d175 as hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/hbase/meta/1588230740/table/87d9eb8168774d27a18dd25f4d13d175
2024-11-17T12:46:46,282 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/hbase/meta/1588230740/table/87d9eb8168774d27a18dd25f4d13d175, entries=2, sequenceid=11, filesize=5.3 K
2024-11-17T12:46:46,284 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 174ms, sequenceid=11, compaction requested=false
2024-11-17T12:46:46,295 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-17T12:46:46,297 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T12:46:46,297 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T12:46:46,297 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731847606109Running coprocessor pre-close hooks at 1731847606109Disabling compacts and flushes for region at 1731847606109Disabling writes for close at 1731847606109Obtaining lock to block concurrent updates at 1731847606110 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731847606110Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1731847606110Flushing stores of hbase:meta,,1.1588230740 at 1731847606111 (+1 ms)Flushing 1588230740/info: creating writer at 1731847606111Flushing 1588230740/info: appending metadata at 1731847606133 (+22 ms)Flushing 1588230740/info: closing flushed file at 1731847606133Flushing 1588230740/ns: creating writer at 1731847606150 (+17 ms)Flushing 1588230740/ns: appending metadata at 1731847606168 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731847606169 (+1 ms)Flushing 1588230740/table: creating writer at 1731847606197 (+28 ms)Flushing 1588230740/table: appending metadata at 1731847606214 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731847606214Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57702062: reopening flushed file at 1731847606233 (+19 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74fbe8c7: reopening flushed file at 1731847606248 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46855a40: reopening flushed file at 1731847606270 (+22 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 174ms, sequenceid=11, compaction requested=false at 1731847606284 (+14 ms)Writing region close event to WAL at 1731847606286 (+2 ms)Running coprocessor post-close hooks at 1731847606297 (+11 ms)Closed at 1731847606297
2024-11-17T12:46:46,297 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-17T12:46:46,310 INFO  [RS:0;0d751a622496:39415 {}] regionserver.HRegionServer(976): stopping server 0d751a622496,39415,1731847510969; all regions closed.
2024-11-17T12:46:46,311 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:46,311 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:46,312 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:46,312 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:46,312 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:46,316 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741834_1010 (size=3066)
2024-11-17T12:46:46,319 DEBUG [RS:0;0d751a622496:39415 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/oldWALs
2024-11-17T12:46:46,319 INFO  [RS:0;0d751a622496:39415 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0d751a622496%2C39415%2C1731847510969.meta:.meta(num 1731847512935)
2024-11-17T12:46:46,319 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741834_1010 (size=3066)
2024-11-17T12:46:46,320 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:46,320 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:46,320 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:46,320 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:46,320 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:46,323 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741847_1023 (size=12695)
2024-11-17T12:46:46,323 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741847_1023 (size=12695)
2024-11-17T12:46:46,327 DEBUG [RS:0;0d751a622496:39415 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/oldWALs
2024-11-17T12:46:46,327 INFO  [RS:0;0d751a622496:39415 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0d751a622496%2C39415%2C1731847510969:(num 1731847585999)
2024-11-17T12:46:46,327 DEBUG [RS:0;0d751a622496:39415 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:46:46,327 INFO  [RS:0;0d751a622496:39415 {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T12:46:46,327 INFO  [RS:0;0d751a622496:39415 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T12:46:46,328 INFO  [RS:0;0d751a622496:39415 {}] hbase.ChoreService(370): Chore service for: regionserver/0d751a622496:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-17T12:46:46,328 INFO  [RS:0;0d751a622496:39415 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T12:46:46,329 INFO  [RS:0;0d751a622496:39415 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39415
2024-11-17T12:46:46,330 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T12:46:46,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T12:46:46,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x1014a1d67db0001, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0d751a622496,39415,1731847510969
2024-11-17T12:46:46,345 INFO  [RS:0;0d751a622496:39415 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T12:46:46,346 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0d751a622496,39415,1731847510969]
2024-11-17T12:46:46,362 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0d751a622496,39415,1731847510969 already deleted, retry=false
2024-11-17T12:46:46,363 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0d751a622496,39415,1731847510969 expired; onlineServers=0
2024-11-17T12:46:46,363 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0d751a622496,42805,1731847510113' *****
2024-11-17T12:46:46,363 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-17T12:46:46,363 INFO  [M:0;0d751a622496:42805 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T12:46:46,363 INFO  [M:0;0d751a622496:42805 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T12:46:46,363 DEBUG [M:0;0d751a622496:42805 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-17T12:46:46,363 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-17T12:46:46,363 DEBUG [M:0;0d751a622496:42805 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-17T12:46:46,363 DEBUG [master/0d751a622496:0:becomeActiveMaster-HFileCleaner.large.0-1731847512155 {}] cleaner.HFileCleaner(306): Exit Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.large.0-1731847512155,5,FailOnTimeoutGroup]
2024-11-17T12:46:46,364 DEBUG [master/0d751a622496:0:becomeActiveMaster-HFileCleaner.small.0-1731847512156 {}] cleaner.HFileCleaner(306): Exit Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.small.0-1731847512156,5,FailOnTimeoutGroup]
2024-11-17T12:46:46,364 INFO  [M:0;0d751a622496:42805 {}] hbase.ChoreService(370): Chore service for: master/0d751a622496:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-17T12:46:46,364 INFO  [M:0;0d751a622496:42805 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T12:46:46,364 DEBUG [M:0;0d751a622496:42805 {}] master.HMaster(1795): Stopping service threads
2024-11-17T12:46:46,364 INFO  [M:0;0d751a622496:42805 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-17T12:46:46,364 INFO  [M:0;0d751a622496:42805 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T12:46:46,365 INFO  [M:0;0d751a622496:42805 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-17T12:46:46,365 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-17T12:46:46,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-17T12:46:46,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:46,371 DEBUG [M:0;0d751a622496:42805 {}] zookeeper.ZKUtil(347): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-17T12:46:46,371 WARN  [M:0;0d751a622496:42805 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-17T12:46:46,372 INFO  [M:0;0d751a622496:42805 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/.lastflushedseqids
2024-11-17T12:46:46,384 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741854_1030 (size=130)
2024-11-17T12:46:46,384 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741854_1030 (size=130)
2024-11-17T12:46:46,385 INFO  [M:0;0d751a622496:42805 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-17T12:46:46,385 INFO  [M:0;0d751a622496:42805 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-17T12:46:46,385 DEBUG [M:0;0d751a622496:42805 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T12:46:46,385 INFO  [M:0;0d751a622496:42805 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:46:46,385 DEBUG [M:0;0d751a622496:42805 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:46:46,385 DEBUG [M:0;0d751a622496:42805 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T12:46:46,385 DEBUG [M:0;0d751a622496:42805 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:46:46,386 INFO  [M:0;0d751a622496:42805 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.04 KB heapSize=29.21 KB
2024-11-17T12:46:46,404 DEBUG [M:0;0d751a622496:42805 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3f40aaf8ded046f88849dd286bdd49d2 is 82, key is hbase:meta,,1/info:regioninfo/1731847513003/Put/seqid=0
2024-11-17T12:46:46,414 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741855_1031 (size=5672)
2024-11-17T12:46:46,414 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741855_1031 (size=5672)
2024-11-17T12:46:46,415 INFO  [M:0;0d751a622496:42805 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3f40aaf8ded046f88849dd286bdd49d2
2024-11-17T12:46:46,419 INFO  [regionserver/0d751a622496:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T12:46:46,446 DEBUG [M:0;0d751a622496:42805 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a6f5baaf777c4ac785ad3634ea902472 is 767, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731847513898/Put/seqid=0
2024-11-17T12:46:46,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x1014a1d67db0001, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:46:46,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39415-0x1014a1d67db0001, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:46:46,455 INFO  [RS:0;0d751a622496:39415 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T12:46:46,455 INFO  [RS:0;0d751a622496:39415 {}] regionserver.HRegionServer(1031): Exiting; stopping=0d751a622496,39415,1731847510969; zookeeper connection closed.
2024-11-17T12:46:46,456 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2ca51d03 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2ca51d03
2024-11-17T12:46:46,456 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-17T12:46:46,457 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741856_1032 (size=6248)
2024-11-17T12:46:46,457 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741856_1032 (size=6248)
2024-11-17T12:46:46,458 INFO  [M:0;0d751a622496:42805 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.43 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a6f5baaf777c4ac785ad3634ea902472
2024-11-17T12:46:46,465 INFO  [M:0;0d751a622496:42805 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a6f5baaf777c4ac785ad3634ea902472
2024-11-17T12:46:46,488 DEBUG [M:0;0d751a622496:42805 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5400f1caa0fc43058e718adc2a5ab295 is 69, key is 0d751a622496,39415,1731847510969/rs:state/1731847512263/Put/seqid=0
2024-11-17T12:46:46,503 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741857_1033 (size=5156)
2024-11-17T12:46:46,506 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741857_1033 (size=5156)
2024-11-17T12:46:46,506 INFO  [M:0;0d751a622496:42805 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5400f1caa0fc43058e718adc2a5ab295
2024-11-17T12:46:46,535 DEBUG [M:0;0d751a622496:42805 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/8cccdba6c1fb4e5984f4880e54982768 is 52, key is load_balancer_on/state:d/1731847513369/Put/seqid=0
2024-11-17T12:46:46,544 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741858_1034 (size=5056)
2024-11-17T12:46:46,545 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741858_1034 (size=5056)
2024-11-17T12:46:46,545 INFO  [M:0;0d751a622496:42805 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/8cccdba6c1fb4e5984f4880e54982768
2024-11-17T12:46:46,556 DEBUG [M:0;0d751a622496:42805 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3f40aaf8ded046f88849dd286bdd49d2 as hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3f40aaf8ded046f88849dd286bdd49d2
2024-11-17T12:46:46,567 INFO  [M:0;0d751a622496:42805 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3f40aaf8ded046f88849dd286bdd49d2, entries=8, sequenceid=59, filesize=5.5 K
2024-11-17T12:46:46,568 DEBUG [M:0;0d751a622496:42805 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a6f5baaf777c4ac785ad3634ea902472 as hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a6f5baaf777c4ac785ad3634ea902472
2024-11-17T12:46:46,581 INFO  [M:0;0d751a622496:42805 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a6f5baaf777c4ac785ad3634ea902472
2024-11-17T12:46:46,581 INFO  [M:0;0d751a622496:42805 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a6f5baaf777c4ac785ad3634ea902472, entries=6, sequenceid=59, filesize=6.1 K
2024-11-17T12:46:46,583 DEBUG [M:0;0d751a622496:42805 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5400f1caa0fc43058e718adc2a5ab295 as hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5400f1caa0fc43058e718adc2a5ab295
2024-11-17T12:46:46,592 INFO  [M:0;0d751a622496:42805 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5400f1caa0fc43058e718adc2a5ab295, entries=1, sequenceid=59, filesize=5.0 K
2024-11-17T12:46:46,594 DEBUG [M:0;0d751a622496:42805 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/8cccdba6c1fb4e5984f4880e54982768 as hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/8cccdba6c1fb4e5984f4880e54982768
2024-11-17T12:46:46,605 INFO  [M:0;0d751a622496:42805 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/8cccdba6c1fb4e5984f4880e54982768, entries=1, sequenceid=59, filesize=4.9 K
2024-11-17T12:46:46,607 INFO  [M:0;0d751a622496:42805 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.04 KB/23588, heapSize ~29.15 KB/29848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 222ms, sequenceid=59, compaction requested=false
2024-11-17T12:46:46,611 INFO  [M:0;0d751a622496:42805 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:46:46,611 DEBUG [M:0;0d751a622496:42805 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731847606385Disabling compacts and flushes for region at 1731847606385Disabling writes for close at 1731847606385Obtaining lock to block concurrent updates at 1731847606386 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731847606386Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23588, getHeapSize=29848, getOffHeapSize=0, getCellsCount=70 at 1731847606386Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731847606387 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731847606387Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731847606404 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731847606404Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731847606423 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731847606445 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731847606445Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731847606465 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731847606487 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731847606487Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731847606517 (+30 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731847606535 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731847606535Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5db34811: reopening flushed file at 1731847606555 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49518690: reopening flushed file at 1731847606567 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6737745a: reopening flushed file at 1731847606581 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72918df1: reopening flushed file at 1731847606593 (+12 ms)Finished flush of dataSize ~23.04 KB/23588, heapSize ~29.15 KB/29848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 222ms, sequenceid=59, compaction requested=false at 1731847606607 (+14 ms)Writing region close event to WAL at 1731847606611 (+4 ms)Closed at 1731847606611
2024-11-17T12:46:46,612 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:46,612 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:46,612 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:46,612 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:46,612 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:46,615 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36729 is added to blk_1073741830_1006 (size=27985)
2024-11-17T12:46:46,615 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43907 is added to blk_1073741830_1006 (size=27985)
2024-11-17T12:46:46,616 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T12:46:46,617 INFO  [M:0;0d751a622496:42805 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-17T12:46:46,617 INFO  [M:0;0d751a622496:42805 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42805
2024-11-17T12:46:46,618 INFO  [M:0;0d751a622496:42805 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T12:46:46,746 INFO  [M:0;0d751a622496:42805 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T12:46:46,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:46:46,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42805-0x1014a1d67db0000, quorum=127.0.0.1:55241, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:46:46,751 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bf97579{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:46:46,753 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:46:46,754 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:46:46,754 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:46:46,754 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/hadoop.log.dir/,STOPPED}
2024-11-17T12:46:46,758 WARN  [BP-362034389-172.17.0.2-1731847506610 heartbeating to localhost/127.0.0.1:38459 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T12:46:46,758 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T12:46:46,759 WARN  [BP-362034389-172.17.0.2-1731847506610 heartbeating to localhost/127.0.0.1:38459 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-362034389-172.17.0.2-1731847506610 (Datanode Uuid 4648dfcf-5b9f-4a8c-84a5-b40fdbefd53c) service to localhost/127.0.0.1:38459
2024-11-17T12:46:46,759 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T12:46:46,760 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/cluster_1ccbeaaa-4148-7ce0-570b-22872a7cb186/data/data3/current/BP-362034389-172.17.0.2-1731847506610 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:46:46,760 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/cluster_1ccbeaaa-4148-7ce0-570b-22872a7cb186/data/data4/current/BP-362034389-172.17.0.2-1731847506610 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:46:46,761 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T12:46:46,766 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7b07d1ba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:46:46,766 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:46:46,767 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:46:46,767 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:46:46,767 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/hadoop.log.dir/,STOPPED}
2024-11-17T12:46:46,769 WARN  [BP-362034389-172.17.0.2-1731847506610 heartbeating to localhost/127.0.0.1:38459 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T12:46:46,769 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T12:46:46,769 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T12:46:46,769 WARN  [BP-362034389-172.17.0.2-1731847506610 heartbeating to localhost/127.0.0.1:38459 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-362034389-172.17.0.2-1731847506610 (Datanode Uuid 6c2e2e5a-ea18-4b82-b022-c156cf4c2479) service to localhost/127.0.0.1:38459
2024-11-17T12:46:46,770 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/cluster_1ccbeaaa-4148-7ce0-570b-22872a7cb186/data/data1/current/BP-362034389-172.17.0.2-1731847506610 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:46:46,771 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/cluster_1ccbeaaa-4148-7ce0-570b-22872a7cb186/data/data2/current/BP-362034389-172.17.0.2-1731847506610 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:46:46,772 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T12:46:46,787 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@735fa16a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T12:46:46,788 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:46:46,788 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:46:46,789 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:46:46,789 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/hadoop.log.dir/,STOPPED}
2024-11-17T12:46:46,801 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-17T12:46:46,847 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-17T12:46:46,857 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=77 (was 12)
Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-4-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: ForkJoinPool-2-worker-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: nioEventLoopGroup-2-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38459
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)

Potentially hanging thread: SSL Certificates Store Monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Time-limited test.named-queue-events-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
	app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
	app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
	app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:38459
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: nioEventLoopGroup-4-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-5-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: master/0d751a622496:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SnapshotHandlerChoreCleaner
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38459 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: SessionTracker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:38459
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@2977aefe
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
	app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
	app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38459
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: region-location-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38459 from jenkins.hfs.0
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-2-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38459
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Async-Client-Retry-Timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/0d751a622496:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcClient-timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38459 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-4-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: master/0d751a622496:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=404 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=205 (was 248), ProcessCount=11 (was 11), AvailableMemoryMB=6719 (was 6722)
2024-11-17T12:46:46,863 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=78, OpenFileDescriptor=404, MaxFileDescriptor=1048576, SystemLoadAverage=205, ProcessCount=11, AvailableMemoryMB=6718
2024-11-17T12:46:46,864 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-17T12:46:46,864 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/hadoop.log.dir so I do NOT create it in target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300
2024-11-17T12:46:46,864 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72abe20d-ed34-1564-7720-50cc8cbc63c1/hadoop.tmp.dir so I do NOT create it in target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300
2024-11-17T12:46:46,864 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/cluster_b7e5f0fa-a691-2ae2-74b8-ed66ab7dc8a2, deleteOnExit=true
2024-11-17T12:46:46,864 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-17T12:46:46,865 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/test.cache.data in system properties and HBase conf
2024-11-17T12:46:46,865 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/hadoop.tmp.dir in system properties and HBase conf
2024-11-17T12:46:46,865 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/hadoop.log.dir in system properties and HBase conf
2024-11-17T12:46:46,865 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-17T12:46:46,865 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-17T12:46:46,865 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-17T12:46:46,865 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-17T12:46:46,866 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-17T12:46:46,866 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-17T12:46:46,866 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-17T12:46:46,866 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T12:46:46,866 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-17T12:46:46,866 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-17T12:46:46,866 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T12:46:46,866 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T12:46:46,866 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-17T12:46:46,866 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/nfs.dump.dir in system properties and HBase conf
2024-11-17T12:46:46,866 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/java.io.tmpdir in system properties and HBase conf
2024-11-17T12:46:46,866 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T12:46:46,867 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-17T12:46:46,867 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-17T12:46:46,881 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T12:46:47,170 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:46:47,177 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:46:47,181 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:46:47,182 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:46:47,182 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-17T12:46:47,182 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:46:47,183 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75cbfab9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:46:47,183 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2f841e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:46:47,286 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@51b7943b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/java.io.tmpdir/jetty-localhost-37269-hadoop-hdfs-3_4_1-tests_jar-_-any-7677285267569561096/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T12:46:47,286 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@15ba4d19{HTTP/1.1, (http/1.1)}{localhost:37269}
2024-11-17T12:46:47,286 INFO  [Time-limited test {}] server.Server(415): Started @102528ms
2024-11-17T12:46:47,299 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T12:46:47,542 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:46:47,546 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:46:47,547 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:46:47,547 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:46:47,547 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T12:46:47,548 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3aa9354f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:46:47,548 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3e23c0c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:46:47,647 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@45b9e849{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/java.io.tmpdir/jetty-localhost-43909-hadoop-hdfs-3_4_1-tests_jar-_-any-13547701027483780623/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:46:47,648 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7d119060{HTTP/1.1, (http/1.1)}{localhost:43909}
2024-11-17T12:46:47,648 INFO  [Time-limited test {}] server.Server(415): Started @102889ms
2024-11-17T12:46:47,649 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T12:46:47,701 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:46:47,707 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:46:47,707 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:46:47,708 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:46:47,708 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-17T12:46:47,708 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1bc8c098{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:46:47,709 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1ce533a5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:46:47,818 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@e721f3d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/java.io.tmpdir/jetty-localhost-33243-hadoop-hdfs-3_4_1-tests_jar-_-any-16160752336217780580/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:46:47,819 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2ff5ef6c{HTTP/1.1, (http/1.1)}{localhost:33243}
2024-11-17T12:46:47,819 INFO  [Time-limited test {}] server.Server(415): Started @103061ms
2024-11-17T12:46:47,820 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T12:46:48,483 WARN  [Thread-453 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/cluster_b7e5f0fa-a691-2ae2-74b8-ed66ab7dc8a2/data/data2/current/BP-1873436019-172.17.0.2-1731847606894/current, will proceed with Du for space computation calculation, 
2024-11-17T12:46:48,483 WARN  [Thread-452 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/cluster_b7e5f0fa-a691-2ae2-74b8-ed66ab7dc8a2/data/data1/current/BP-1873436019-172.17.0.2-1731847606894/current, will proceed with Du for space computation calculation, 
2024-11-17T12:46:48,503 WARN  [Thread-416 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T12:46:48,506 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x129466162f3f881 with lease ID 0x22b94eee134a20a6: Processing first storage report for DS-916f423c-d1ed-499e-ac50-976c5fc895d7 from datanode DatanodeRegistration(127.0.0.1:46403, datanodeUuid=f4938294-2374-4246-9bb4-2b5a763af214, infoPort=45765, infoSecurePort=0, ipcPort=39281, storageInfo=lv=-57;cid=testClusterID;nsid=916983231;c=1731847606894)
2024-11-17T12:46:48,507 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x129466162f3f881 with lease ID 0x22b94eee134a20a6: from storage DS-916f423c-d1ed-499e-ac50-976c5fc895d7 node DatanodeRegistration(127.0.0.1:46403, datanodeUuid=f4938294-2374-4246-9bb4-2b5a763af214, infoPort=45765, infoSecurePort=0, ipcPort=39281, storageInfo=lv=-57;cid=testClusterID;nsid=916983231;c=1731847606894), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:46:48,507 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x129466162f3f881 with lease ID 0x22b94eee134a20a6: Processing first storage report for DS-61d28fa2-a781-4805-b1f8-6c38ce88ed00 from datanode DatanodeRegistration(127.0.0.1:46403, datanodeUuid=f4938294-2374-4246-9bb4-2b5a763af214, infoPort=45765, infoSecurePort=0, ipcPort=39281, storageInfo=lv=-57;cid=testClusterID;nsid=916983231;c=1731847606894)
2024-11-17T12:46:48,507 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x129466162f3f881 with lease ID 0x22b94eee134a20a6: from storage DS-61d28fa2-a781-4805-b1f8-6c38ce88ed00 node DatanodeRegistration(127.0.0.1:46403, datanodeUuid=f4938294-2374-4246-9bb4-2b5a763af214, infoPort=45765, infoSecurePort=0, ipcPort=39281, storageInfo=lv=-57;cid=testClusterID;nsid=916983231;c=1731847606894), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:46:48,673 WARN  [Thread-463 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/cluster_b7e5f0fa-a691-2ae2-74b8-ed66ab7dc8a2/data/data3/current/BP-1873436019-172.17.0.2-1731847606894/current, will proceed with Du for space computation calculation, 
2024-11-17T12:46:48,673 WARN  [Thread-464 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/cluster_b7e5f0fa-a691-2ae2-74b8-ed66ab7dc8a2/data/data4/current/BP-1873436019-172.17.0.2-1731847606894/current, will proceed with Du for space computation calculation, 
2024-11-17T12:46:48,694 WARN  [Thread-439 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T12:46:48,697 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xed3ababb3076532b with lease ID 0x22b94eee134a20a7: Processing first storage report for DS-042f96f0-89e0-4a2e-ae3e-900461e0fc8c from datanode DatanodeRegistration(127.0.0.1:42581, datanodeUuid=a2074768-c49d-43ee-a492-ef89f5c1eb2c, infoPort=38107, infoSecurePort=0, ipcPort=32973, storageInfo=lv=-57;cid=testClusterID;nsid=916983231;c=1731847606894)
2024-11-17T12:46:48,697 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xed3ababb3076532b with lease ID 0x22b94eee134a20a7: from storage DS-042f96f0-89e0-4a2e-ae3e-900461e0fc8c node DatanodeRegistration(127.0.0.1:42581, datanodeUuid=a2074768-c49d-43ee-a492-ef89f5c1eb2c, infoPort=38107, infoSecurePort=0, ipcPort=32973, storageInfo=lv=-57;cid=testClusterID;nsid=916983231;c=1731847606894), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:46:48,697 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xed3ababb3076532b with lease ID 0x22b94eee134a20a7: Processing first storage report for DS-aba40a66-71c8-4ac1-8bd4-ad4c4267b717 from datanode DatanodeRegistration(127.0.0.1:42581, datanodeUuid=a2074768-c49d-43ee-a492-ef89f5c1eb2c, infoPort=38107, infoSecurePort=0, ipcPort=32973, storageInfo=lv=-57;cid=testClusterID;nsid=916983231;c=1731847606894)
2024-11-17T12:46:48,697 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xed3ababb3076532b with lease ID 0x22b94eee134a20a7: from storage DS-aba40a66-71c8-4ac1-8bd4-ad4c4267b717 node DatanodeRegistration(127.0.0.1:42581, datanodeUuid=a2074768-c49d-43ee-a492-ef89f5c1eb2c, infoPort=38107, infoSecurePort=0, ipcPort=32973, storageInfo=lv=-57;cid=testClusterID;nsid=916983231;c=1731847606894), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:46:48,756 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300
2024-11-17T12:46:48,760 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/cluster_b7e5f0fa-a691-2ae2-74b8-ed66ab7dc8a2/zookeeper_0, clientPort=58448, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/cluster_b7e5f0fa-a691-2ae2-74b8-ed66ab7dc8a2/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/cluster_b7e5f0fa-a691-2ae2-74b8-ed66ab7dc8a2/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-17T12:46:48,761 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58448
2024-11-17T12:46:48,762 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:46:48,764 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:46:48,775 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46403 is added to blk_1073741825_1001 (size=7)
2024-11-17T12:46:48,775 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42581 is added to blk_1073741825_1001 (size=7)
2024-11-17T12:46:48,777 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8 with version=8
2024-11-17T12:46:48,777 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/hbase-staging
2024-11-17T12:46:48,780 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/0d751a622496:0 server-side Connection retries=45
2024-11-17T12:46:48,780 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:46:48,780 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T12:46:48,780 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T12:46:48,780 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:46:48,780 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T12:46:48,781 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-17T12:46:48,781 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T12:46:48,781 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33389
2024-11-17T12:46:48,783 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33389 connecting to ZooKeeper ensemble=127.0.0.1:58448
2024-11-17T12:46:48,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:333890x0, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T12:46:48,825 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33389-0x1014a1eec7c0000 connected
2024-11-17T12:46:48,907 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:46:48,909 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:46:48,911 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:46:48,912 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8, hbase.cluster.distributed=false
2024-11-17T12:46:48,913 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T12:46:48,914 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33389
2024-11-17T12:46:48,914 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33389
2024-11-17T12:46:48,914 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33389
2024-11-17T12:46:48,916 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33389
2024-11-17T12:46:48,918 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33389
2024-11-17T12:46:48,936 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/0d751a622496:0 server-side Connection retries=45
2024-11-17T12:46:48,936 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:46:48,936 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T12:46:48,936 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T12:46:48,936 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:46:48,937 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T12:46:48,937 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-17T12:46:48,937 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T12:46:48,937 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44685
2024-11-17T12:46:48,939 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44685 connecting to ZooKeeper ensemble=127.0.0.1:58448
2024-11-17T12:46:48,940 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:46:48,942 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:46:48,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:446850x0, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T12:46:48,958 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:446850x0, quorum=127.0.0.1:58448, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:46:48,958 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44685-0x1014a1eec7c0001 connected
2024-11-17T12:46:48,958 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-17T12:46:48,959 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-17T12:46:48,960 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44685-0x1014a1eec7c0001, quorum=127.0.0.1:58448, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-17T12:46:48,961 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44685-0x1014a1eec7c0001, quorum=127.0.0.1:58448, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T12:46:48,962 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44685
2024-11-17T12:46:48,963 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44685
2024-11-17T12:46:48,967 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44685
2024-11-17T12:46:48,970 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44685
2024-11-17T12:46:48,974 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44685
2024-11-17T12:46:48,992 DEBUG [M:0;0d751a622496:33389 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0d751a622496:33389
2024-11-17T12:46:48,993 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0d751a622496,33389,1731847608780
2024-11-17T12:46:49,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44685-0x1014a1eec7c0001, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:46:49,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:46:49,005 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0d751a622496,33389,1731847608780
2024-11-17T12:46:49,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:49,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44685-0x1014a1eec7c0001, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-17T12:46:49,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44685-0x1014a1eec7c0001, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:49,017 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-17T12:46:49,017 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0d751a622496,33389,1731847608780 from backup master directory
2024-11-17T12:46:49,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44685-0x1014a1eec7c0001, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:46:49,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0d751a622496,33389,1731847608780
2024-11-17T12:46:49,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:46:49,029 WARN  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T12:46:49,029 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0d751a622496,33389,1731847608780
2024-11-17T12:46:49,036 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/hbase.id] with ID: 2d8a07d4-26b7-43a4-9fb1-32eb0f72f95a
2024-11-17T12:46:49,036 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/.tmp/hbase.id
2024-11-17T12:46:49,045 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42581 is added to blk_1073741826_1002 (size=42)
2024-11-17T12:46:49,046 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46403 is added to blk_1073741826_1002 (size=42)
2024-11-17T12:46:49,048 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/.tmp/hbase.id]:[hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/hbase.id]
2024-11-17T12:46:49,069 INFO  [master/0d751a622496:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:46:49,070 INFO  [master/0d751a622496:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-17T12:46:49,072 INFO  [master/0d751a622496:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-17T12:46:49,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44685-0x1014a1eec7c0001, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:49,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:49,094 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46403 is added to blk_1073741827_1003 (size=196)
2024-11-17T12:46:49,094 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42581 is added to blk_1073741827_1003 (size=196)
2024-11-17T12:46:49,095 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-17T12:46:49,096 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-17T12:46:49,097 INFO  [master/0d751a622496:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T12:46:49,106 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46403 is added to blk_1073741828_1004 (size=1189)
2024-11-17T12:46:49,106 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42581 is added to blk_1073741828_1004 (size=1189)
2024-11-17T12:46:49,108 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/data/master/store
2024-11-17T12:46:49,118 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42581 is added to blk_1073741829_1005 (size=34)
2024-11-17T12:46:49,118 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46403 is added to blk_1073741829_1005 (size=34)
2024-11-17T12:46:49,119 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:46:49,119 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T12:46:49,119 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:46:49,119 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:46:49,119 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T12:46:49,119 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:46:49,119 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:46:49,119 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731847609119Disabling compacts and flushes for region at 1731847609119Disabling writes for close at 1731847609119Writing region close event to WAL at 1731847609119Closed at 1731847609119
2024-11-17T12:46:49,120 WARN  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/data/master/store/.initializing
2024-11-17T12:46:49,120 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/WALs/0d751a622496,33389,1731847608780
2024-11-17T12:46:49,123 INFO  [master/0d751a622496:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C33389%2C1731847608780, suffix=, logDir=hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/WALs/0d751a622496,33389,1731847608780, archiveDir=hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/oldWALs, maxLogs=10
2024-11-17T12:46:49,123 INFO  [master/0d751a622496:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C33389%2C1731847608780.1731847609123
2024-11-17T12:46:49,129 INFO  [master/0d751a622496:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/WALs/0d751a622496,33389,1731847608780/0d751a622496%2C33389%2C1731847608780.1731847609123
2024-11-17T12:46:49,129 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38107:38107),(127.0.0.1/127.0.0.1:45765:45765)]
2024-11-17T12:46:49,130 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-17T12:46:49,130 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:46:49,131 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:46:49,131 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:46:49,132 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:46:49,134 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-17T12:46:49,134 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:49,135 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:46:49,135 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:46:49,137 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-17T12:46:49,137 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:49,137 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:46:49,137 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:46:49,139 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-17T12:46:49,140 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:49,140 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:46:49,140 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:46:49,142 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-17T12:46:49,142 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:49,142 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:46:49,142 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:46:49,143 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:46:49,144 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:46:49,145 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:46:49,145 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:46:49,146 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-17T12:46:49,147 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:46:49,151 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T12:46:49,152 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=871484, jitterRate=0.10814978182315826}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-17T12:46:49,153 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731847609131Initializing all the Stores at 1731847609132 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847609132Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847609132Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847609132Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847609132Cleaning up temporary data from old regions at 1731847609145 (+13 ms)Region opened successfully at 1731847609153 (+8 ms)
2024-11-17T12:46:49,153 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-17T12:46:49,157 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c00c36f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0d751a622496/172.17.0.2:0
2024-11-17T12:46:49,158 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-17T12:46:49,159 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-17T12:46:49,159 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-17T12:46:49,159 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-17T12:46:49,160 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-17T12:46:49,160 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-17T12:46:49,160 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-17T12:46:49,165 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-17T12:46:49,166 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-17T12:46:49,199 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-17T12:46:49,199 INFO  [master/0d751a622496:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-17T12:46:49,200 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-17T12:46:49,207 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-17T12:46:49,207 INFO  [master/0d751a622496:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-17T12:46:49,208 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-17T12:46:49,215 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-17T12:46:49,216 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-17T12:46:49,223 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-17T12:46:49,226 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-17T12:46:49,237 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-17T12:46:49,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44685-0x1014a1eec7c0001, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T12:46:49,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T12:46:49,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:49,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44685-0x1014a1eec7c0001, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:49,251 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0d751a622496,33389,1731847608780, sessionid=0x1014a1eec7c0000, setting cluster-up flag (Was=false)
2024-11-17T12:46:49,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44685-0x1014a1eec7c0001, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:49,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:49,299 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-17T12:46:49,300 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0d751a622496,33389,1731847608780
2024-11-17T12:46:49,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44685-0x1014a1eec7c0001, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:49,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:49,349 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-17T12:46:49,350 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0d751a622496,33389,1731847608780
2024-11-17T12:46:49,351 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-17T12:46:49,353 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-17T12:46:49,353 INFO  [master/0d751a622496:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-17T12:46:49,354 INFO  [master/0d751a622496:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-17T12:46:49,354 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0d751a622496,33389,1731847608780
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-17T12:46:49,355 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:46:49,355 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:46:49,356 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:46:49,356 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:46:49,356 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0d751a622496:0, corePoolSize=10, maxPoolSize=10
2024-11-17T12:46:49,356 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:49,356 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0d751a622496:0, corePoolSize=2, maxPoolSize=2
2024-11-17T12:46:49,356 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:49,357 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731847639357
2024-11-17T12:46:49,357 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-17T12:46:49,357 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-17T12:46:49,358 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-17T12:46:49,358 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-17T12:46:49,358 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-17T12:46:49,358 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T12:46:49,358 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-17T12:46:49,358 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-17T12:46:49,359 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:49,359 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-17T12:46:49,362 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:49,364 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-17T12:46:49,364 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-17T12:46:49,364 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-17T12:46:49,364 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-17T12:46:49,365 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-17T12:46:49,365 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.large.0-1731847609365,5,FailOnTimeoutGroup]
2024-11-17T12:46:49,365 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.small.0-1731847609365,5,FailOnTimeoutGroup]
2024-11-17T12:46:49,365 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:49,365 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-17T12:46:49,365 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:49,366 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:49,370 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42581 is added to blk_1073741831_1007 (size=1321)
2024-11-17T12:46:49,370 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46403 is added to blk_1073741831_1007 (size=1321)
2024-11-17T12:46:49,371 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-17T12:46:49,372 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8
2024-11-17T12:46:49,378 INFO  [RS:0;0d751a622496:44685 {}] regionserver.HRegionServer(746): ClusterId : 2d8a07d4-26b7-43a4-9fb1-32eb0f72f95a
2024-11-17T12:46:49,378 DEBUG [RS:0;0d751a622496:44685 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-17T12:46:49,383 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46403 is added to blk_1073741832_1008 (size=32)
2024-11-17T12:46:49,383 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42581 is added to blk_1073741832_1008 (size=32)
2024-11-17T12:46:49,384 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:46:49,387 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T12:46:49,388 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T12:46:49,388 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:49,389 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:46:49,389 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T12:46:49,391 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T12:46:49,391 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:49,392 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:46:49,392 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T12:46:49,394 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T12:46:49,394 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:49,395 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:46:49,395 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T12:46:49,396 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T12:46:49,396 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:49,397 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:46:49,397 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T12:46:49,398 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/data/hbase/meta/1588230740
2024-11-17T12:46:49,399 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/data/hbase/meta/1588230740
2024-11-17T12:46:49,400 DEBUG [RS:0;0d751a622496:44685 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-17T12:46:49,400 DEBUG [RS:0;0d751a622496:44685 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-17T12:46:49,401 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T12:46:49,401 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T12:46:49,402 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T12:46:49,403 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T12:46:49,406 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T12:46:49,407 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=841140, jitterRate=0.06956490874290466}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T12:46:49,409 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731847609384Initializing all the Stores at 1731847609385 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847609385Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847609386 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847609386Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847609386Cleaning up temporary data from old regions at 1731847609401 (+15 ms)Region opened successfully at 1731847609408 (+7 ms)
2024-11-17T12:46:49,409 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T12:46:49,409 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T12:46:49,409 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T12:46:49,409 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T12:46:49,409 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T12:46:49,410 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T12:46:49,410 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731847609409Disabling compacts and flushes for region at 1731847609409Disabling writes for close at 1731847609409Writing region close event to WAL at 1731847609410 (+1 ms)Closed at 1731847609410
2024-11-17T12:46:49,411 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T12:46:49,412 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-17T12:46:49,412 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-17T12:46:49,414 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T12:46:49,415 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-17T12:46:49,416 DEBUG [RS:0;0d751a622496:44685 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-17T12:46:49,416 DEBUG [RS:0;0d751a622496:44685 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c49638a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0d751a622496/172.17.0.2:0
2024-11-17T12:46:49,429 DEBUG [RS:0;0d751a622496:44685 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0d751a622496:44685
2024-11-17T12:46:49,430 INFO  [RS:0;0d751a622496:44685 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-17T12:46:49,430 INFO  [RS:0;0d751a622496:44685 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-17T12:46:49,430 DEBUG [RS:0;0d751a622496:44685 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-17T12:46:49,431 INFO  [RS:0;0d751a622496:44685 {}] regionserver.HRegionServer(2659): reportForDuty to master=0d751a622496,33389,1731847608780 with port=44685, startcode=1731847608936
2024-11-17T12:46:49,431 DEBUG [RS:0;0d751a622496:44685 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-17T12:46:49,433 INFO  [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55597, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-11-17T12:46:49,433 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33389 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0d751a622496,44685,1731847608936
2024-11-17T12:46:49,434 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33389 {}] master.ServerManager(517): Registering regionserver=0d751a622496,44685,1731847608936
2024-11-17T12:46:49,436 DEBUG [RS:0;0d751a622496:44685 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8
2024-11-17T12:46:49,436 DEBUG [RS:0;0d751a622496:44685 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44873
2024-11-17T12:46:49,436 DEBUG [RS:0;0d751a622496:44685 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-17T12:46:49,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T12:46:49,446 DEBUG [RS:0;0d751a622496:44685 {}] zookeeper.ZKUtil(111): regionserver:44685-0x1014a1eec7c0001, quorum=127.0.0.1:58448, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0d751a622496,44685,1731847608936
2024-11-17T12:46:49,446 WARN  [RS:0;0d751a622496:44685 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T12:46:49,446 INFO  [RS:0;0d751a622496:44685 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T12:46:49,446 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0d751a622496,44685,1731847608936]
2024-11-17T12:46:49,446 DEBUG [RS:0;0d751a622496:44685 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/WALs/0d751a622496,44685,1731847608936
2024-11-17T12:46:49,451 INFO  [RS:0;0d751a622496:44685 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-17T12:46:49,454 INFO  [RS:0;0d751a622496:44685 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-17T12:46:49,454 INFO  [RS:0;0d751a622496:44685 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-17T12:46:49,454 INFO  [RS:0;0d751a622496:44685 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:49,455 INFO  [RS:0;0d751a622496:44685 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-17T12:46:49,456 INFO  [RS:0;0d751a622496:44685 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-17T12:46:49,456 INFO  [RS:0;0d751a622496:44685 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:49,456 DEBUG [RS:0;0d751a622496:44685 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:49,456 DEBUG [RS:0;0d751a622496:44685 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:49,456 DEBUG [RS:0;0d751a622496:44685 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:49,456 DEBUG [RS:0;0d751a622496:44685 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:49,457 DEBUG [RS:0;0d751a622496:44685 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:49,457 DEBUG [RS:0;0d751a622496:44685 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0, corePoolSize=2, maxPoolSize=2
2024-11-17T12:46:49,457 DEBUG [RS:0;0d751a622496:44685 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:49,457 DEBUG [RS:0;0d751a622496:44685 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:49,457 DEBUG [RS:0;0d751a622496:44685 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:49,457 DEBUG [RS:0;0d751a622496:44685 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:49,457 DEBUG [RS:0;0d751a622496:44685 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:49,457 DEBUG [RS:0;0d751a622496:44685 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:49,457 DEBUG [RS:0;0d751a622496:44685 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0d751a622496:0, corePoolSize=3, maxPoolSize=3
2024-11-17T12:46:49,458 DEBUG [RS:0;0d751a622496:44685 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0, corePoolSize=3, maxPoolSize=3
2024-11-17T12:46:49,462 INFO  [RS:0;0d751a622496:44685 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:49,463 INFO  [RS:0;0d751a622496:44685 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:49,463 INFO  [RS:0;0d751a622496:44685 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:49,463 INFO  [RS:0;0d751a622496:44685 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:49,463 INFO  [RS:0;0d751a622496:44685 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:49,463 INFO  [RS:0;0d751a622496:44685 {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,44685,1731847608936-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T12:46:49,480 INFO  [RS:0;0d751a622496:44685 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-17T12:46:49,480 INFO  [RS:0;0d751a622496:44685 {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,44685,1731847608936-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:49,480 INFO  [RS:0;0d751a622496:44685 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:49,481 INFO  [RS:0;0d751a622496:44685 {}] regionserver.Replication(171): 0d751a622496,44685,1731847608936 started
2024-11-17T12:46:49,496 INFO  [RS:0;0d751a622496:44685 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:49,496 INFO  [RS:0;0d751a622496:44685 {}] regionserver.HRegionServer(1482): Serving as 0d751a622496,44685,1731847608936, RpcServer on 0d751a622496/172.17.0.2:44685, sessionid=0x1014a1eec7c0001
2024-11-17T12:46:49,496 DEBUG [RS:0;0d751a622496:44685 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-17T12:46:49,496 DEBUG [RS:0;0d751a622496:44685 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0d751a622496,44685,1731847608936
2024-11-17T12:46:49,496 DEBUG [RS:0;0d751a622496:44685 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0d751a622496,44685,1731847608936'
2024-11-17T12:46:49,496 DEBUG [RS:0;0d751a622496:44685 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-17T12:46:49,497 DEBUG [RS:0;0d751a622496:44685 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-17T12:46:49,498 DEBUG [RS:0;0d751a622496:44685 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-17T12:46:49,498 DEBUG [RS:0;0d751a622496:44685 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-17T12:46:49,498 DEBUG [RS:0;0d751a622496:44685 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0d751a622496,44685,1731847608936
2024-11-17T12:46:49,498 DEBUG [RS:0;0d751a622496:44685 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0d751a622496,44685,1731847608936'
2024-11-17T12:46:49,498 DEBUG [RS:0;0d751a622496:44685 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-17T12:46:49,499 DEBUG [RS:0;0d751a622496:44685 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-17T12:46:49,499 DEBUG [RS:0;0d751a622496:44685 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-17T12:46:49,500 INFO  [RS:0;0d751a622496:44685 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-17T12:46:49,500 INFO  [RS:0;0d751a622496:44685 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-17T12:46:49,565 WARN  [0d751a622496:33389 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-17T12:46:49,602 INFO  [RS:0;0d751a622496:44685 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C44685%2C1731847608936, suffix=, logDir=hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/WALs/0d751a622496,44685,1731847608936, archiveDir=hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/oldWALs, maxLogs=32
2024-11-17T12:46:49,604 INFO  [RS:0;0d751a622496:44685 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C44685%2C1731847608936.1731847609604
2024-11-17T12:46:49,613 INFO  [RS:0;0d751a622496:44685 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/WALs/0d751a622496,44685,1731847608936/0d751a622496%2C44685%2C1731847608936.1731847609604
2024-11-17T12:46:49,615 DEBUG [RS:0;0d751a622496:44685 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45765:45765),(127.0.0.1/127.0.0.1:38107:38107)]
2024-11-17T12:46:49,815 DEBUG [0d751a622496:33389 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-17T12:46:49,816 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0d751a622496,44685,1731847608936
2024-11-17T12:46:49,818 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0d751a622496,44685,1731847608936, state=OPENING
2024-11-17T12:46:49,832 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-17T12:46:49,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:49,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44685-0x1014a1eec7c0001, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:49,841 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T12:46:49,842 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:46:49,842 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0d751a622496,44685,1731847608936}]
2024-11-17T12:46:49,842 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:46:49,998 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-17T12:46:50,001 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49017, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-17T12:46:50,006 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-17T12:46:50,006 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T12:46:50,009 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C44685%2C1731847608936.meta, suffix=.meta, logDir=hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/WALs/0d751a622496,44685,1731847608936, archiveDir=hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/oldWALs, maxLogs=32
2024-11-17T12:46:50,012 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C44685%2C1731847608936.meta.1731847610011.meta
2024-11-17T12:46:50,026 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/WALs/0d751a622496,44685,1731847608936/0d751a622496%2C44685%2C1731847608936.meta.1731847610011.meta
2024-11-17T12:46:50,030 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45765:45765),(127.0.0.1/127.0.0.1:38107:38107)]
2024-11-17T12:46:50,034 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-17T12:46:50,034 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-17T12:46:50,034 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-17T12:46:50,034 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-17T12:46:50,034 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-17T12:46:50,034 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:46:50,035 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-17T12:46:50,035 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-17T12:46:50,037 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T12:46:50,039 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T12:46:50,039 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:50,040 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:46:50,040 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T12:46:50,042 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T12:46:50,042 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:50,043 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:46:50,043 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T12:46:50,044 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T12:46:50,044 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:50,045 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:46:50,045 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T12:46:50,046 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T12:46:50,046 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:50,047 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:46:50,047 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T12:46:50,048 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/data/hbase/meta/1588230740
2024-11-17T12:46:50,050 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/data/hbase/meta/1588230740
2024-11-17T12:46:50,052 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T12:46:50,052 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T12:46:50,053 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T12:46:50,056 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T12:46:50,057 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=811033, jitterRate=0.03128272294998169}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T12:46:50,057 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-17T12:46:50,058 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731847610035Writing region info on filesystem at 1731847610035Initializing all the Stores at 1731847610036 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847610036Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847610037 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847610037Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847610037Cleaning up temporary data from old regions at 1731847610052 (+15 ms)Running coprocessor post-open hooks at 1731847610057 (+5 ms)Region opened successfully at 1731847610058 (+1 ms)
2024-11-17T12:46:50,060 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731847609998
2024-11-17T12:46:50,063 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-17T12:46:50,063 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-17T12:46:50,065 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0d751a622496,44685,1731847608936
2024-11-17T12:46:50,066 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0d751a622496,44685,1731847608936, state=OPEN
2024-11-17T12:46:50,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44685-0x1014a1eec7c0001, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T12:46:50,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T12:46:50,122 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0d751a622496,44685,1731847608936
2024-11-17T12:46:50,122 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:46:50,122 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:46:50,128 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-17T12:46:50,128 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0d751a622496,44685,1731847608936 in 280 msec
2024-11-17T12:46:50,131 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-17T12:46:50,131 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 716 msec
2024-11-17T12:46:50,132 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T12:46:50,133 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-17T12:46:50,135 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T12:46:50,135 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0d751a622496,44685,1731847608936, seqNum=-1]
2024-11-17T12:46:50,135 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T12:46:50,137 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33653, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T12:46:50,146 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 792 msec
2024-11-17T12:46:50,146 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731847610146, completionTime=-1
2024-11-17T12:46:50,146 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-17T12:46:50,146 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-17T12:46:50,149 INFO  [master/0d751a622496:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-17T12:46:50,149 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731847670149
2024-11-17T12:46:50,149 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731847730149
2024-11-17T12:46:50,149 INFO  [master/0d751a622496:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-17T12:46:50,150 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,33389,1731847608780-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:50,150 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,33389,1731847608780-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:50,150 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,33389,1731847608780-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:50,150 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0d751a622496:33389, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:50,150 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:50,150 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:50,153 DEBUG [master/0d751a622496:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-17T12:46:50,158 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.128sec
2024-11-17T12:46:50,158 INFO  [master/0d751a622496:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-17T12:46:50,158 INFO  [master/0d751a622496:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-17T12:46:50,158 INFO  [master/0d751a622496:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-17T12:46:50,158 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-17T12:46:50,158 INFO  [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-17T12:46:50,158 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,33389,1731847608780-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T12:46:50,158 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,33389,1731847608780-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-17T12:46:50,166 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-17T12:46:50,166 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-17T12:46:50,166 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,33389,1731847608780-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:50,183 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bd69191, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T12:46:50,183 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0d751a622496,33389,-1 for getting cluster id
2024-11-17T12:46:50,183 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-17T12:46:50,186 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2d8a07d4-26b7-43a4-9fb1-32eb0f72f95a'
2024-11-17T12:46:50,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-17T12:46:50,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2d8a07d4-26b7-43a4-9fb1-32eb0f72f95a"

2024-11-17T12:46:50,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56abadfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T12:46:50,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0d751a622496,33389,-1]
2024-11-17T12:46:50,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-17T12:46:50,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:46:50,192 INFO  [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-17T12:46:50,194 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e299140, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T12:46:50,195 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T12:46:50,196 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0d751a622496,44685,1731847608936, seqNum=-1]
2024-11-17T12:46:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T12:46:50,200 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T12:46:50,201 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0d751a622496,33389,1731847608780
2024-11-17T12:46:50,202 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:46:50,206 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-17T12:46:50,206 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-17T12:46:50,206 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T12:46:50,207 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T12:46:50,207 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:46:50,207 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:46:50,207 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-17T12:46:50,207 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-17T12:46:50,207 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=538851360, stopped=false
2024-11-17T12:46:50,207 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0d751a622496,33389,1731847608780
2024-11-17T12:46:50,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44685-0x1014a1eec7c0001, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T12:46:50,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T12:46:50,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44685-0x1014a1eec7c0001, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:50,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:50,224 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T12:46:50,224 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T12:46:50,224 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T12:46:50,224 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:46:50,224 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:46:50,224 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44685-0x1014a1eec7c0001, quorum=127.0.0.1:58448, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:46:50,224 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0d751a622496,44685,1731847608936' *****
2024-11-17T12:46:50,225 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-17T12:46:50,225 INFO  [RS:0;0d751a622496:44685 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-17T12:46:50,225 INFO  [RS:0;0d751a622496:44685 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-17T12:46:50,225 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-17T12:46:50,225 INFO  [RS:0;0d751a622496:44685 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-17T12:46:50,225 INFO  [RS:0;0d751a622496:44685 {}] regionserver.HRegionServer(959): stopping server 0d751a622496,44685,1731847608936
2024-11-17T12:46:50,225 INFO  [RS:0;0d751a622496:44685 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T12:46:50,225 INFO  [RS:0;0d751a622496:44685 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0d751a622496:44685.
2024-11-17T12:46:50,225 DEBUG [RS:0;0d751a622496:44685 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T12:46:50,225 DEBUG [RS:0;0d751a622496:44685 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:46:50,226 INFO  [RS:0;0d751a622496:44685 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-17T12:46:50,226 INFO  [RS:0;0d751a622496:44685 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-17T12:46:50,226 INFO  [RS:0;0d751a622496:44685 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-17T12:46:50,226 INFO  [RS:0;0d751a622496:44685 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-17T12:46:50,226 INFO  [RS:0;0d751a622496:44685 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-17T12:46:50,226 DEBUG [RS:0;0d751a622496:44685 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-17T12:46:50,226 DEBUG [RS:0;0d751a622496:44685 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-17T12:46:50,226 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T12:46:50,226 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T12:46:50,226 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T12:46:50,226 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T12:46:50,226 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T12:46:50,227 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-17T12:46:50,246 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/data/hbase/meta/1588230740/.tmp/ns/7a88f2a6c43e46c4aef3c1d4cee83f5d is 43, key is default/ns:d/1731847610138/Put/seqid=0
2024-11-17T12:46:50,252 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42581 is added to blk_1073741835_1011 (size=5153)
2024-11-17T12:46:50,252 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46403 is added to blk_1073741835_1011 (size=5153)
2024-11-17T12:46:50,252 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/data/hbase/meta/1588230740/.tmp/ns/7a88f2a6c43e46c4aef3c1d4cee83f5d
2024-11-17T12:46:50,263 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/data/hbase/meta/1588230740/.tmp/ns/7a88f2a6c43e46c4aef3c1d4cee83f5d as hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/data/hbase/meta/1588230740/ns/7a88f2a6c43e46c4aef3c1d4cee83f5d
2024-11-17T12:46:50,272 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/data/hbase/meta/1588230740/ns/7a88f2a6c43e46c4aef3c1d4cee83f5d, entries=2, sequenceid=6, filesize=5.0 K
2024-11-17T12:46:50,274 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 47ms, sequenceid=6, compaction requested=false
2024-11-17T12:46:50,274 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-17T12:46:50,281 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-17T12:46:50,282 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T12:46:50,282 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T12:46:50,283 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731847610226Running coprocessor pre-close hooks at 1731847610226Disabling compacts and flushes for region at 1731847610226Disabling writes for close at 1731847610226Obtaining lock to block concurrent updates at 1731847610227 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731847610227Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1731847610227Flushing stores of hbase:meta,,1.1588230740 at 1731847610228 (+1 ms)Flushing 1588230740/ns: creating writer at 1731847610228Flushing 1588230740/ns: appending metadata at 1731847610245 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731847610245Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3dcd33f3: reopening flushed file at 1731847610261 (+16 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 47ms, sequenceid=6, compaction requested=false at 1731847610274 (+13 ms)Writing region close event to WAL at 1731847610276 (+2 ms)Running coprocessor post-close hooks at 1731847610282 (+6 ms)Closed at 1731847610282
2024-11-17T12:46:50,283 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-17T12:46:50,426 INFO  [RS:0;0d751a622496:44685 {}] regionserver.HRegionServer(976): stopping server 0d751a622496,44685,1731847608936; all regions closed.
2024-11-17T12:46:50,427 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:50,427 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:50,427 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:50,428 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:50,428 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:50,430 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42581 is added to blk_1073741834_1010 (size=1152)
2024-11-17T12:46:50,431 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46403 is added to blk_1073741834_1010 (size=1152)
2024-11-17T12:46:50,434 DEBUG [RS:0;0d751a622496:44685 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/oldWALs
2024-11-17T12:46:50,434 INFO  [RS:0;0d751a622496:44685 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0d751a622496%2C44685%2C1731847608936.meta:.meta(num 1731847610011)
2024-11-17T12:46:50,434 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:50,435 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:50,435 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:50,435 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:50,435 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:50,438 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42581 is added to blk_1073741833_1009 (size=93)
2024-11-17T12:46:50,438 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46403 is added to blk_1073741833_1009 (size=93)
2024-11-17T12:46:50,441 DEBUG [RS:0;0d751a622496:44685 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/oldWALs
2024-11-17T12:46:50,441 INFO  [RS:0;0d751a622496:44685 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0d751a622496%2C44685%2C1731847608936:(num 1731847609604)
2024-11-17T12:46:50,441 DEBUG [RS:0;0d751a622496:44685 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:46:50,441 INFO  [RS:0;0d751a622496:44685 {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T12:46:50,441 INFO  [RS:0;0d751a622496:44685 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T12:46:50,442 INFO  [RS:0;0d751a622496:44685 {}] hbase.ChoreService(370): Chore service for: regionserver/0d751a622496:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-17T12:46:50,442 INFO  [RS:0;0d751a622496:44685 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T12:46:50,442 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T12:46:50,442 INFO  [RS:0;0d751a622496:44685 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44685
2024-11-17T12:46:50,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T12:46:50,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44685-0x1014a1eec7c0001, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0d751a622496,44685,1731847608936
2024-11-17T12:46:50,457 INFO  [RS:0;0d751a622496:44685 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T12:46:50,457 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0d751a622496,44685,1731847608936]
2024-11-17T12:46:50,473 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0d751a622496,44685,1731847608936 already deleted, retry=false
2024-11-17T12:46:50,473 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0d751a622496,44685,1731847608936 expired; onlineServers=0
2024-11-17T12:46:50,473 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0d751a622496,33389,1731847608780' *****
2024-11-17T12:46:50,473 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-17T12:46:50,473 INFO  [M:0;0d751a622496:33389 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T12:46:50,474 INFO  [M:0;0d751a622496:33389 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T12:46:50,474 DEBUG [M:0;0d751a622496:33389 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-17T12:46:50,474 DEBUG [M:0;0d751a622496:33389 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-17T12:46:50,474 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-17T12:46:50,474 DEBUG [master/0d751a622496:0:becomeActiveMaster-HFileCleaner.small.0-1731847609365 {}] cleaner.HFileCleaner(306): Exit Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.small.0-1731847609365,5,FailOnTimeoutGroup]
2024-11-17T12:46:50,474 DEBUG [master/0d751a622496:0:becomeActiveMaster-HFileCleaner.large.0-1731847609365 {}] cleaner.HFileCleaner(306): Exit Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.large.0-1731847609365,5,FailOnTimeoutGroup]
2024-11-17T12:46:50,474 INFO  [M:0;0d751a622496:33389 {}] hbase.ChoreService(370): Chore service for: master/0d751a622496:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-17T12:46:50,474 INFO  [M:0;0d751a622496:33389 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T12:46:50,474 DEBUG [M:0;0d751a622496:33389 {}] master.HMaster(1795): Stopping service threads
2024-11-17T12:46:50,474 INFO  [M:0;0d751a622496:33389 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-17T12:46:50,474 INFO  [M:0;0d751a622496:33389 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T12:46:50,474 INFO  [M:0;0d751a622496:33389 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-17T12:46:50,475 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-17T12:46:50,479 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T12:46:50,480 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-17T12:46:50,481 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-17T12:46:50,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-17T12:46:50,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:50,482 DEBUG [M:0;0d751a622496:33389 {}] zookeeper.ZKUtil(347): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-17T12:46:50,482 WARN  [M:0;0d751a622496:33389 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-17T12:46:50,482 INFO  [M:0;0d751a622496:33389 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/.lastflushedseqids
2024-11-17T12:46:50,489 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42581 is added to blk_1073741836_1012 (size=108)
2024-11-17T12:46:50,489 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46403 is added to blk_1073741836_1012 (size=108)
2024-11-17T12:46:50,490 INFO  [M:0;0d751a622496:33389 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-17T12:46:50,490 INFO  [M:0;0d751a622496:33389 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-17T12:46:50,490 DEBUG [M:0;0d751a622496:33389 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T12:46:50,490 INFO  [M:0;0d751a622496:33389 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:46:50,491 DEBUG [M:0;0d751a622496:33389 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:46:50,491 DEBUG [M:0;0d751a622496:33389 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T12:46:50,491 DEBUG [M:0;0d751a622496:33389 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:46:50,491 INFO  [M:0;0d751a622496:33389 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-17T12:46:50,512 DEBUG [M:0;0d751a622496:33389 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a3126ecd45c14f62b7bec426e07f7e76 is 82, key is hbase:meta,,1/info:regioninfo/1731847610064/Put/seqid=0
2024-11-17T12:46:50,520 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46403 is added to blk_1073741837_1013 (size=5672)
2024-11-17T12:46:50,521 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42581 is added to blk_1073741837_1013 (size=5672)
2024-11-17T12:46:50,522 INFO  [M:0;0d751a622496:33389 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a3126ecd45c14f62b7bec426e07f7e76
2024-11-17T12:46:50,552 DEBUG [M:0;0d751a622496:33389 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a4d6fc6ae796480d9cc3a85bdd991ca1 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1731847610145/Put/seqid=0
2024-11-17T12:46:50,559 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46403 is added to blk_1073741838_1014 (size=5275)
2024-11-17T12:46:50,559 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42581 is added to blk_1073741838_1014 (size=5275)
2024-11-17T12:46:50,560 INFO  [M:0;0d751a622496:33389 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a4d6fc6ae796480d9cc3a85bdd991ca1
2024-11-17T12:46:50,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44685-0x1014a1eec7c0001, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:46:50,565 INFO  [RS:0;0d751a622496:44685 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T12:46:50,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44685-0x1014a1eec7c0001, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:46:50,565 INFO  [RS:0;0d751a622496:44685 {}] regionserver.HRegionServer(1031): Exiting; stopping=0d751a622496,44685,1731847608936; zookeeper connection closed.
2024-11-17T12:46:50,566 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@ab0bec2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@ab0bec2
2024-11-17T12:46:50,567 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-17T12:46:50,593 DEBUG [M:0;0d751a622496:33389 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f995cab4460c4761945c8015219c69ba is 69, key is 0d751a622496,44685,1731847608936/rs:state/1731847609434/Put/seqid=0
2024-11-17T12:46:50,599 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46403 is added to blk_1073741839_1015 (size=5156)
2024-11-17T12:46:50,600 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42581 is added to blk_1073741839_1015 (size=5156)
2024-11-17T12:46:50,600 INFO  [M:0;0d751a622496:33389 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f995cab4460c4761945c8015219c69ba
2024-11-17T12:46:50,639 DEBUG [M:0;0d751a622496:33389 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6dd311cc3bf44d16912def7cc706e7f4 is 52, key is load_balancer_on/state:d/1731847610205/Put/seqid=0
2024-11-17T12:46:50,648 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42581 is added to blk_1073741840_1016 (size=5056)
2024-11-17T12:46:50,648 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46403 is added to blk_1073741840_1016 (size=5056)
2024-11-17T12:46:50,651 INFO  [M:0;0d751a622496:33389 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6dd311cc3bf44d16912def7cc706e7f4
2024-11-17T12:46:50,662 DEBUG [M:0;0d751a622496:33389 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a3126ecd45c14f62b7bec426e07f7e76 as hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a3126ecd45c14f62b7bec426e07f7e76
2024-11-17T12:46:50,673 INFO  [M:0;0d751a622496:33389 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a3126ecd45c14f62b7bec426e07f7e76, entries=8, sequenceid=29, filesize=5.5 K
2024-11-17T12:46:50,675 DEBUG [M:0;0d751a622496:33389 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a4d6fc6ae796480d9cc3a85bdd991ca1 as hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a4d6fc6ae796480d9cc3a85bdd991ca1
2024-11-17T12:46:50,684 INFO  [M:0;0d751a622496:33389 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a4d6fc6ae796480d9cc3a85bdd991ca1, entries=3, sequenceid=29, filesize=5.2 K
2024-11-17T12:46:50,686 DEBUG [M:0;0d751a622496:33389 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f995cab4460c4761945c8015219c69ba as hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f995cab4460c4761945c8015219c69ba
2024-11-17T12:46:50,696 INFO  [M:0;0d751a622496:33389 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f995cab4460c4761945c8015219c69ba, entries=1, sequenceid=29, filesize=5.0 K
2024-11-17T12:46:50,698 DEBUG [M:0;0d751a622496:33389 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6dd311cc3bf44d16912def7cc706e7f4 as hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6dd311cc3bf44d16912def7cc706e7f4
2024-11-17T12:46:50,709 INFO  [M:0;0d751a622496:33389 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44873/user/jenkins/test-data/e15a6bd8-96b4-4cae-c3cc-3b0fe4edb1f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6dd311cc3bf44d16912def7cc706e7f4, entries=1, sequenceid=29, filesize=4.9 K
2024-11-17T12:46:50,710 INFO  [M:0;0d751a622496:33389 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 219ms, sequenceid=29, compaction requested=false
2024-11-17T12:46:50,715 INFO  [M:0;0d751a622496:33389 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:46:50,715 DEBUG [M:0;0d751a622496:33389 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731847610490Disabling compacts and flushes for region at 1731847610490Disabling writes for close at 1731847610491 (+1 ms)Obtaining lock to block concurrent updates at 1731847610491Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731847610491Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1731847610492 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731847610493 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731847610493Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731847610512 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731847610512Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731847610529 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731847610551 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731847610551Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731847610570 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731847610592 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731847610592Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731847610609 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731847610638 (+29 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731847610638Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a09f5b1: reopening flushed file at 1731847610659 (+21 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f3e60fb: reopening flushed file at 1731847610674 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5df0b550: reopening flushed file at 1731847610684 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28da68a8: reopening flushed file at 1731847610696 (+12 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 219ms, sequenceid=29, compaction requested=false at 1731847610710 (+14 ms)Writing region close event to WAL at 1731847610715 (+5 ms)Closed at 1731847610715
2024-11-17T12:46:50,716 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:50,716 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:50,716 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:50,716 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:50,716 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:46:50,719 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46403 is added to blk_1073741830_1006 (size=10311)
2024-11-17T12:46:50,720 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42581 is added to blk_1073741830_1006 (size=10311)
2024-11-17T12:46:50,723 INFO  [M:0;0d751a622496:33389 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-17T12:46:50,723 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T12:46:50,724 INFO  [M:0;0d751a622496:33389 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33389
2024-11-17T12:46:50,725 INFO  [M:0;0d751a622496:33389 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T12:46:50,840 INFO  [M:0;0d751a622496:33389 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T12:46:50,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:46:50,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33389-0x1014a1eec7c0000, quorum=127.0.0.1:58448, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:46:50,843 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@e721f3d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:46:50,843 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2ff5ef6c{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:46:50,843 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:46:50,843 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1ce533a5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:46:50,843 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1bc8c098{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/hadoop.log.dir/,STOPPED}
2024-11-17T12:46:50,844 WARN  [BP-1873436019-172.17.0.2-1731847606894 heartbeating to localhost/127.0.0.1:44873 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T12:46:50,844 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T12:46:50,844 WARN  [BP-1873436019-172.17.0.2-1731847606894 heartbeating to localhost/127.0.0.1:44873 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1873436019-172.17.0.2-1731847606894 (Datanode Uuid a2074768-c49d-43ee-a492-ef89f5c1eb2c) service to localhost/127.0.0.1:44873
2024-11-17T12:46:50,844 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T12:46:50,845 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/cluster_b7e5f0fa-a691-2ae2-74b8-ed66ab7dc8a2/data/data3/current/BP-1873436019-172.17.0.2-1731847606894 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:46:50,845 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/cluster_b7e5f0fa-a691-2ae2-74b8-ed66ab7dc8a2/data/data4/current/BP-1873436019-172.17.0.2-1731847606894 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:46:50,845 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T12:46:50,854 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@45b9e849{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:46:50,855 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7d119060{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:46:50,855 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:46:50,855 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3e23c0c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:46:50,856 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3aa9354f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/hadoop.log.dir/,STOPPED}
2024-11-17T12:46:50,858 WARN  [BP-1873436019-172.17.0.2-1731847606894 heartbeating to localhost/127.0.0.1:44873 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T12:46:50,858 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T12:46:50,858 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T12:46:50,858 WARN  [BP-1873436019-172.17.0.2-1731847606894 heartbeating to localhost/127.0.0.1:44873 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1873436019-172.17.0.2-1731847606894 (Datanode Uuid f4938294-2374-4246-9bb4-2b5a763af214) service to localhost/127.0.0.1:44873
2024-11-17T12:46:50,858 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/cluster_b7e5f0fa-a691-2ae2-74b8-ed66ab7dc8a2/data/data1/current/BP-1873436019-172.17.0.2-1731847606894 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:46:50,859 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/cluster_b7e5f0fa-a691-2ae2-74b8-ed66ab7dc8a2/data/data2/current/BP-1873436019-172.17.0.2-1731847606894 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:46:50,859 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T12:46:50,865 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@51b7943b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T12:46:50,866 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@15ba4d19{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:46:50,866 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:46:50,867 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2f841e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:46:50,867 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75cbfab9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/hadoop.log.dir/,STOPPED}
2024-11-17T12:46:50,873 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-17T12:46:50,898 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-17T12:46:50,898 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-17T12:46:50,898 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/hadoop.log.dir so I do NOT create it in target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361
2024-11-17T12:46:50,898 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b7c00909-20cb-7f0c-d027-d1752e074300/hadoop.tmp.dir so I do NOT create it in target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361
2024-11-17T12:46:50,898 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676, deleteOnExit=true
2024-11-17T12:46:50,898 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-17T12:46:50,899 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/test.cache.data in system properties and HBase conf
2024-11-17T12:46:50,899 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/hadoop.tmp.dir in system properties and HBase conf
2024-11-17T12:46:50,899 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/hadoop.log.dir in system properties and HBase conf
2024-11-17T12:46:50,899 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-17T12:46:50,899 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-17T12:46:50,899 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-17T12:46:50,900 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-17T12:46:50,900 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-17T12:46:50,900 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-17T12:46:50,901 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-17T12:46:50,901 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T12:46:50,901 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-17T12:46:50,901 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-17T12:46:50,901 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T12:46:50,901 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T12:46:50,901 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-17T12:46:50,901 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/nfs.dump.dir in system properties and HBase conf
2024-11-17T12:46:50,901 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/java.io.tmpdir in system properties and HBase conf
2024-11-17T12:46:50,901 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T12:46:50,901 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-17T12:46:50,902 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-17T12:46:50,922 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T12:46:51,201 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:46:51,203 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:46:51,207 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:46:51,209 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:46:51,211 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:46:51,211 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:46:51,211 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T12:46:51,212 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:46:51,213 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ab5393f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:46:51,213 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4ac253d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:46:51,310 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2295376c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/java.io.tmpdir/jetty-localhost-34219-hadoop-hdfs-3_4_1-tests_jar-_-any-15049370898173794914/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T12:46:51,311 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@54adbc26{HTTP/1.1, (http/1.1)}{localhost:34219}
2024-11-17T12:46:51,311 INFO  [Time-limited test {}] server.Server(415): Started @106552ms
2024-11-17T12:46:51,323 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T12:46:51,463 INFO  [regionserver/0d751a622496:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T12:46:51,512 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-17T12:46:51,513 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:46:51,528 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:46:51,529 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:46:51,530 WARN  [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:46:51,540 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:46:51,543 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:46:51,544 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:46:51,544 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:46:51,544 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-17T12:46:51,545 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@48d478e7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:46:51,545 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7d4a4bdf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:46:51,638 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@71ad97f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/java.io.tmpdir/jetty-localhost-41537-hadoop-hdfs-3_4_1-tests_jar-_-any-14291677094906044561/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:46:51,639 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3f14f219{HTTP/1.1, (http/1.1)}{localhost:41537}
2024-11-17T12:46:51,639 INFO  [Time-limited test {}] server.Server(415): Started @106880ms
2024-11-17T12:46:51,640 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T12:46:51,679 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:46:51,684 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:46:51,691 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:46:51,691 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:46:51,691 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T12:46:51,691 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@10a4d310{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:46:51,692 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@73b17c7e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:46:51,791 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2ade06e3{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/java.io.tmpdir/jetty-localhost-42347-hadoop-hdfs-3_4_1-tests_jar-_-any-7583027629362075261/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:46:51,792 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1baf7059{HTTP/1.1, (http/1.1)}{localhost:42347}
2024-11-17T12:46:51,792 INFO  [Time-limited test {}] server.Server(415): Started @107033ms
2024-11-17T12:46:51,793 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T12:46:52,308 WARN  [Thread-671 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data1/current/BP-1311131906-172.17.0.2-1731847610939/current, will proceed with Du for space computation calculation, 
2024-11-17T12:46:52,309 WARN  [Thread-672 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data2/current/BP-1311131906-172.17.0.2-1731847610939/current, will proceed with Du for space computation calculation, 
2024-11-17T12:46:52,331 WARN  [Thread-636 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T12:46:52,333 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc2f45a8a7490494e with lease ID 0xfc001fc3e009519: Processing first storage report for DS-a138e678-6576-4789-8707-5ef24a4fc7ac from datanode DatanodeRegistration(127.0.0.1:45021, datanodeUuid=158584b4-ac14-402d-849d-a116eafa11ed, infoPort=38513, infoSecurePort=0, ipcPort=33121, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939)
2024-11-17T12:46:52,334 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc2f45a8a7490494e with lease ID 0xfc001fc3e009519: from storage DS-a138e678-6576-4789-8707-5ef24a4fc7ac node DatanodeRegistration(127.0.0.1:45021, datanodeUuid=158584b4-ac14-402d-849d-a116eafa11ed, infoPort=38513, infoSecurePort=0, ipcPort=33121, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:46:52,334 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc2f45a8a7490494e with lease ID 0xfc001fc3e009519: Processing first storage report for DS-190e5970-26c5-4233-8ea1-f856c3f98b70 from datanode DatanodeRegistration(127.0.0.1:45021, datanodeUuid=158584b4-ac14-402d-849d-a116eafa11ed, infoPort=38513, infoSecurePort=0, ipcPort=33121, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939)
2024-11-17T12:46:52,334 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc2f45a8a7490494e with lease ID 0xfc001fc3e009519: from storage DS-190e5970-26c5-4233-8ea1-f856c3f98b70 node DatanodeRegistration(127.0.0.1:45021, datanodeUuid=158584b4-ac14-402d-849d-a116eafa11ed, infoPort=38513, infoSecurePort=0, ipcPort=33121, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:46:52,501 WARN  [Thread-683 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data3/current/BP-1311131906-172.17.0.2-1731847610939/current, will proceed with Du for space computation calculation, 
2024-11-17T12:46:52,501 WARN  [Thread-684 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data4/current/BP-1311131906-172.17.0.2-1731847610939/current, will proceed with Du for space computation calculation, 
2024-11-17T12:46:52,516 WARN  [Thread-659 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T12:46:52,519 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbc94c1673493fa03 with lease ID 0xfc001fc3e00951a: Processing first storage report for DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b from datanode DatanodeRegistration(127.0.0.1:33741, datanodeUuid=368a0fe1-98fb-4e11-a7ad-3861a4d4f891, infoPort=34235, infoSecurePort=0, ipcPort=43267, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939)
2024-11-17T12:46:52,519 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbc94c1673493fa03 with lease ID 0xfc001fc3e00951a: from storage DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b node DatanodeRegistration(127.0.0.1:33741, datanodeUuid=368a0fe1-98fb-4e11-a7ad-3861a4d4f891, infoPort=34235, infoSecurePort=0, ipcPort=43267, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:46:52,519 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbc94c1673493fa03 with lease ID 0xfc001fc3e00951a: Processing first storage report for DS-82a38059-b18c-440f-9aea-207b1fa279b2 from datanode DatanodeRegistration(127.0.0.1:33741, datanodeUuid=368a0fe1-98fb-4e11-a7ad-3861a4d4f891, infoPort=34235, infoSecurePort=0, ipcPort=43267, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939)
2024-11-17T12:46:52,519 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbc94c1673493fa03 with lease ID 0xfc001fc3e00951a: from storage DS-82a38059-b18c-440f-9aea-207b1fa279b2 node DatanodeRegistration(127.0.0.1:33741, datanodeUuid=368a0fe1-98fb-4e11-a7ad-3861a4d4f891, infoPort=34235, infoSecurePort=0, ipcPort=43267, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:46:52,532 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361
2024-11-17T12:46:52,536 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/zookeeper_0, clientPort=62721, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-17T12:46:52,537 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62721
2024-11-17T12:46:52,537 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:46:52,539 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:46:52,551 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33741 is added to blk_1073741825_1001 (size=7)
2024-11-17T12:46:52,552 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741825_1001 (size=7)
2024-11-17T12:46:52,553 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8 with version=8
2024-11-17T12:46:52,553 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/hbase-staging
2024-11-17T12:46:52,555 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/0d751a622496:0 server-side Connection retries=45
2024-11-17T12:46:52,555 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:46:52,555 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T12:46:52,556 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T12:46:52,556 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:46:52,556 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T12:46:52,556 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-17T12:46:52,556 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T12:46:52,557 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37969
2024-11-17T12:46:52,559 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37969 connecting to ZooKeeper ensemble=127.0.0.1:62721
2024-11-17T12:46:52,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:379690x0, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T12:46:52,608 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37969-0x1014a1efb3c0000 connected
2024-11-17T12:46:52,681 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:46:52,684 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:46:52,687 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:46:52,687 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8, hbase.cluster.distributed=false
2024-11-17T12:46:52,689 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T12:46:52,690 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37969
2024-11-17T12:46:52,690 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37969
2024-11-17T12:46:52,690 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37969
2024-11-17T12:46:52,691 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37969
2024-11-17T12:46:52,691 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37969
2024-11-17T12:46:52,705 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/0d751a622496:0 server-side Connection retries=45
2024-11-17T12:46:52,706 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:46:52,706 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T12:46:52,706 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T12:46:52,706 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:46:52,706 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T12:46:52,706 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-17T12:46:52,706 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T12:46:52,707 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43029
2024-11-17T12:46:52,708 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43029 connecting to ZooKeeper ensemble=127.0.0.1:62721
2024-11-17T12:46:52,709 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:46:52,711 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:46:52,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:430290x0, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T12:46:52,723 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:430290x0, quorum=127.0.0.1:62721, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:46:52,723 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43029-0x1014a1efb3c0001 connected
2024-11-17T12:46:52,724 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-17T12:46:52,725 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-17T12:46:52,726 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43029-0x1014a1efb3c0001, quorum=127.0.0.1:62721, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-17T12:46:52,727 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43029-0x1014a1efb3c0001, quorum=127.0.0.1:62721, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T12:46:52,730 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43029
2024-11-17T12:46:52,730 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43029
2024-11-17T12:46:52,730 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43029
2024-11-17T12:46:52,731 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43029
2024-11-17T12:46:52,733 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43029
2024-11-17T12:46:52,745 DEBUG [M:0;0d751a622496:37969 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0d751a622496:37969
2024-11-17T12:46:52,745 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0d751a622496,37969,1731847612555
2024-11-17T12:46:52,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43029-0x1014a1efb3c0001, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:46:52,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:46:52,757 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0d751a622496,37969,1731847612555
2024-11-17T12:46:52,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:52,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43029-0x1014a1efb3c0001, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-17T12:46:52,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43029-0x1014a1efb3c0001, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:52,765 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-17T12:46:52,766 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0d751a622496,37969,1731847612555 from backup master directory
2024-11-17T12:46:52,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43029-0x1014a1efb3c0001, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:46:52,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0d751a622496,37969,1731847612555
2024-11-17T12:46:52,773 WARN  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T12:46:52,773 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0d751a622496,37969,1731847612555
2024-11-17T12:46:52,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:46:52,777 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/hbase.id] with ID: eb423186-dfd2-460f-bbe3-1e398e27b24d
2024-11-17T12:46:52,778 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/.tmp/hbase.id
2024-11-17T12:46:52,784 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33741 is added to blk_1073741826_1002 (size=42)
2024-11-17T12:46:52,785 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741826_1002 (size=42)
2024-11-17T12:46:52,785 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/.tmp/hbase.id]:[hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/hbase.id]
2024-11-17T12:46:52,797 INFO  [master/0d751a622496:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:46:52,797 INFO  [master/0d751a622496:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-17T12:46:52,799 INFO  [master/0d751a622496:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-17T12:46:52,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43029-0x1014a1efb3c0001, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:52,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:52,813 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33741 is added to blk_1073741827_1003 (size=196)
2024-11-17T12:46:52,813 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741827_1003 (size=196)
2024-11-17T12:46:52,814 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-17T12:46:52,815 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-17T12:46:52,816 INFO  [master/0d751a622496:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T12:46:52,824 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33741 is added to blk_1073741828_1004 (size=1189)
2024-11-17T12:46:52,825 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741828_1004 (size=1189)
2024-11-17T12:46:52,826 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/data/master/store
2024-11-17T12:46:52,834 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741829_1005 (size=34)
2024-11-17T12:46:52,835 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33741 is added to blk_1073741829_1005 (size=34)
2024-11-17T12:46:52,835 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:46:52,836 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T12:46:52,836 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:46:52,836 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:46:52,836 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T12:46:52,836 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:46:52,836 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:46:52,836 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731847612836Disabling compacts and flushes for region at 1731847612836Disabling writes for close at 1731847612836Writing region close event to WAL at 1731847612836Closed at 1731847612836
2024-11-17T12:46:52,837 WARN  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/data/master/store/.initializing
2024-11-17T12:46:52,837 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/WALs/0d751a622496,37969,1731847612555
2024-11-17T12:46:52,840 INFO  [master/0d751a622496:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C37969%2C1731847612555, suffix=, logDir=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/WALs/0d751a622496,37969,1731847612555, archiveDir=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/oldWALs, maxLogs=10
2024-11-17T12:46:52,841 INFO  [master/0d751a622496:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C37969%2C1731847612555.1731847612841
2024-11-17T12:46:52,846 INFO  [master/0d751a622496:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/WALs/0d751a622496,37969,1731847612555/0d751a622496%2C37969%2C1731847612555.1731847612841
2024-11-17T12:46:52,847 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34235:34235),(127.0.0.1/127.0.0.1:38513:38513)]
2024-11-17T12:46:52,851 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-17T12:46:52,851 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:46:52,852 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:46:52,852 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:46:52,853 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:46:52,855 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-17T12:46:52,855 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:52,856 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:46:52,856 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:46:52,857 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-17T12:46:52,857 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:52,858 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:46:52,858 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:46:52,860 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-17T12:46:52,860 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:52,860 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:46:52,860 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:46:52,861 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-17T12:46:52,861 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:52,862 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:46:52,862 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:46:52,863 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:46:52,863 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:46:52,864 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:46:52,864 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:46:52,865 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-17T12:46:52,866 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:46:52,868 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T12:46:52,869 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=715377, jitterRate=-0.09035137295722961}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-17T12:46:52,870 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731847612852Initializing all the Stores at 1731847612853 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847612853Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847612853Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847612853Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847612853Cleaning up temporary data from old regions at 1731847612864 (+11 ms)Region opened successfully at 1731847612870 (+6 ms)
2024-11-17T12:46:52,870 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-17T12:46:52,873 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4023f934, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0d751a622496/172.17.0.2:0
2024-11-17T12:46:52,875 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-17T12:46:52,875 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-17T12:46:52,875 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-17T12:46:52,875 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-17T12:46:52,875 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-17T12:46:52,876 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-17T12:46:52,876 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-17T12:46:52,878 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-17T12:46:52,879 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-17T12:46:52,906 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-17T12:46:52,906 INFO  [master/0d751a622496:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-17T12:46:52,908 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-17T12:46:52,914 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-17T12:46:52,915 INFO  [master/0d751a622496:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-17T12:46:52,917 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-17T12:46:52,923 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-17T12:46:52,924 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-17T12:46:52,931 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-17T12:46:52,937 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-17T12:46:52,948 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-17T12:46:52,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T12:46:52,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43029-0x1014a1efb3c0001, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T12:46:52,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:52,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43029-0x1014a1efb3c0001, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:52,957 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0d751a622496,37969,1731847612555, sessionid=0x1014a1efb3c0000, setting cluster-up flag (Was=false)
2024-11-17T12:46:52,973 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:52,973 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43029-0x1014a1efb3c0001, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:52,998 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-17T12:46:52,999 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0d751a622496,37969,1731847612555
2024-11-17T12:46:53,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:53,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43029-0x1014a1efb3c0001, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:53,040 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-17T12:46:53,044 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0d751a622496,37969,1731847612555
2024-11-17T12:46:53,047 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-17T12:46:53,050 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-17T12:46:53,051 INFO  [master/0d751a622496:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-17T12:46:53,051 INFO  [master/0d751a622496:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-17T12:46:53,051 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0d751a622496,37969,1731847612555
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-17T12:46:53,053 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:46:53,053 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:46:53,053 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:46:53,054 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:46:53,054 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0d751a622496:0, corePoolSize=10, maxPoolSize=10
2024-11-17T12:46:53,054 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:53,054 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0d751a622496:0, corePoolSize=2, maxPoolSize=2
2024-11-17T12:46:53,054 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:53,059 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731847643059
2024-11-17T12:46:53,059 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-17T12:46:53,059 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-17T12:46:53,060 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-17T12:46:53,060 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-17T12:46:53,060 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-17T12:46:53,060 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-17T12:46:53,060 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:53,060 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T12:46:53,061 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-17T12:46:53,061 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-17T12:46:53,061 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-17T12:46:53,061 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-17T12:46:53,062 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-17T12:46:53,062 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-17T12:46:53,062 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:53,063 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-17T12:46:53,063 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.large.0-1731847613062,5,FailOnTimeoutGroup]
2024-11-17T12:46:53,063 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.small.0-1731847613063,5,FailOnTimeoutGroup]
2024-11-17T12:46:53,063 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:53,063 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-17T12:46:53,064 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:53,064 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:53,071 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741831_1007 (size=1321)
2024-11-17T12:46:53,071 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33741 is added to blk_1073741831_1007 (size=1321)
2024-11-17T12:46:53,072 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-17T12:46:53,072 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8
2024-11-17T12:46:53,080 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741832_1008 (size=32)
2024-11-17T12:46:53,080 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33741 is added to blk_1073741832_1008 (size=32)
2024-11-17T12:46:53,081 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:46:53,082 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T12:46:53,084 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T12:46:53,084 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:53,084 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:46:53,085 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T12:46:53,087 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T12:46:53,087 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:53,087 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:46:53,088 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T12:46:53,090 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T12:46:53,090 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:53,090 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:46:53,091 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T12:46:53,093 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T12:46:53,093 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:53,094 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:46:53,094 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T12:46:53,095 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/hbase/meta/1588230740
2024-11-17T12:46:53,095 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/hbase/meta/1588230740
2024-11-17T12:46:53,097 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T12:46:53,097 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T12:46:53,097 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T12:46:53,098 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T12:46:53,101 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T12:46:53,101 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=874763, jitterRate=0.11231985688209534}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T12:46:53,102 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731847613081Initializing all the Stores at 1731847613082 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847613082Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847613082Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847613082Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847613082Cleaning up temporary data from old regions at 1731847613097 (+15 ms)Region opened successfully at 1731847613102 (+5 ms)
2024-11-17T12:46:53,102 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T12:46:53,102 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T12:46:53,102 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T12:46:53,102 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T12:46:53,102 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T12:46:53,102 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T12:46:53,103 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731847613102Disabling compacts and flushes for region at 1731847613102Disabling writes for close at 1731847613102Writing region close event to WAL at 1731847613102Closed at 1731847613102
2024-11-17T12:46:53,104 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T12:46:53,104 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-17T12:46:53,104 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-17T12:46:53,106 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T12:46:53,107 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-17T12:46:53,135 INFO  [RS:0;0d751a622496:43029 {}] regionserver.HRegionServer(746): ClusterId : eb423186-dfd2-460f-bbe3-1e398e27b24d
2024-11-17T12:46:53,136 DEBUG [RS:0;0d751a622496:43029 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-17T12:46:53,162 DEBUG [RS:0;0d751a622496:43029 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-17T12:46:53,163 DEBUG [RS:0;0d751a622496:43029 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-17T12:46:53,174 DEBUG [RS:0;0d751a622496:43029 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-17T12:46:53,174 DEBUG [RS:0;0d751a622496:43029 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e32cae8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0d751a622496/172.17.0.2:0
2024-11-17T12:46:53,187 DEBUG [RS:0;0d751a622496:43029 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0d751a622496:43029
2024-11-17T12:46:53,187 INFO  [RS:0;0d751a622496:43029 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-17T12:46:53,187 INFO  [RS:0;0d751a622496:43029 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-17T12:46:53,187 DEBUG [RS:0;0d751a622496:43029 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-17T12:46:53,188 INFO  [RS:0;0d751a622496:43029 {}] regionserver.HRegionServer(2659): reportForDuty to master=0d751a622496,37969,1731847612555 with port=43029, startcode=1731847612705
2024-11-17T12:46:53,188 DEBUG [RS:0;0d751a622496:43029 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-17T12:46:53,190 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41283, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-11-17T12:46:53,191 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37969 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0d751a622496,43029,1731847612705
2024-11-17T12:46:53,191 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37969 {}] master.ServerManager(517): Registering regionserver=0d751a622496,43029,1731847612705
2024-11-17T12:46:53,193 DEBUG [RS:0;0d751a622496:43029 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8
2024-11-17T12:46:53,193 DEBUG [RS:0;0d751a622496:43029 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42079
2024-11-17T12:46:53,193 DEBUG [RS:0;0d751a622496:43029 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-17T12:46:53,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T12:46:53,204 DEBUG [RS:0;0d751a622496:43029 {}] zookeeper.ZKUtil(111): regionserver:43029-0x1014a1efb3c0001, quorum=127.0.0.1:62721, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0d751a622496,43029,1731847612705
2024-11-17T12:46:53,204 WARN  [RS:0;0d751a622496:43029 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T12:46:53,204 INFO  [RS:0;0d751a622496:43029 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T12:46:53,204 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0d751a622496,43029,1731847612705]
2024-11-17T12:46:53,204 DEBUG [RS:0;0d751a622496:43029 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705
2024-11-17T12:46:53,208 INFO  [RS:0;0d751a622496:43029 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-17T12:46:53,211 INFO  [RS:0;0d751a622496:43029 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-17T12:46:53,212 INFO  [RS:0;0d751a622496:43029 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-17T12:46:53,212 INFO  [RS:0;0d751a622496:43029 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:53,212 INFO  [RS:0;0d751a622496:43029 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-17T12:46:53,213 INFO  [RS:0;0d751a622496:43029 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-17T12:46:53,214 INFO  [RS:0;0d751a622496:43029 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:53,214 DEBUG [RS:0;0d751a622496:43029 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:53,214 DEBUG [RS:0;0d751a622496:43029 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:53,214 DEBUG [RS:0;0d751a622496:43029 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:53,214 DEBUG [RS:0;0d751a622496:43029 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:53,214 DEBUG [RS:0;0d751a622496:43029 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:53,214 DEBUG [RS:0;0d751a622496:43029 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0, corePoolSize=2, maxPoolSize=2
2024-11-17T12:46:53,214 DEBUG [RS:0;0d751a622496:43029 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:53,214 DEBUG [RS:0;0d751a622496:43029 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:53,214 DEBUG [RS:0;0d751a622496:43029 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:53,214 DEBUG [RS:0;0d751a622496:43029 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:53,214 DEBUG [RS:0;0d751a622496:43029 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:53,214 DEBUG [RS:0;0d751a622496:43029 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:53,215 DEBUG [RS:0;0d751a622496:43029 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0d751a622496:0, corePoolSize=3, maxPoolSize=3
2024-11-17T12:46:53,215 DEBUG [RS:0;0d751a622496:43029 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0, corePoolSize=3, maxPoolSize=3
2024-11-17T12:46:53,215 INFO  [RS:0;0d751a622496:43029 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:53,215 INFO  [RS:0;0d751a622496:43029 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:53,215 INFO  [RS:0;0d751a622496:43029 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:53,215 INFO  [RS:0;0d751a622496:43029 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:53,215 INFO  [RS:0;0d751a622496:43029 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:53,215 INFO  [RS:0;0d751a622496:43029 {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,43029,1731847612705-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T12:46:53,238 INFO  [RS:0;0d751a622496:43029 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-17T12:46:53,238 INFO  [RS:0;0d751a622496:43029 {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,43029,1731847612705-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:53,238 INFO  [RS:0;0d751a622496:43029 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:53,239 INFO  [RS:0;0d751a622496:43029 {}] regionserver.Replication(171): 0d751a622496,43029,1731847612705 started
2024-11-17T12:46:53,257 WARN  [0d751a622496:37969 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-17T12:46:53,261 INFO  [RS:0;0d751a622496:43029 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:53,261 INFO  [RS:0;0d751a622496:43029 {}] regionserver.HRegionServer(1482): Serving as 0d751a622496,43029,1731847612705, RpcServer on 0d751a622496/172.17.0.2:43029, sessionid=0x1014a1efb3c0001
2024-11-17T12:46:53,261 DEBUG [RS:0;0d751a622496:43029 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-17T12:46:53,261 DEBUG [RS:0;0d751a622496:43029 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0d751a622496,43029,1731847612705
2024-11-17T12:46:53,261 DEBUG [RS:0;0d751a622496:43029 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0d751a622496,43029,1731847612705'
2024-11-17T12:46:53,261 DEBUG [RS:0;0d751a622496:43029 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-17T12:46:53,262 DEBUG [RS:0;0d751a622496:43029 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-17T12:46:53,262 DEBUG [RS:0;0d751a622496:43029 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-17T12:46:53,263 DEBUG [RS:0;0d751a622496:43029 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-17T12:46:53,263 DEBUG [RS:0;0d751a622496:43029 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0d751a622496,43029,1731847612705
2024-11-17T12:46:53,263 DEBUG [RS:0;0d751a622496:43029 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0d751a622496,43029,1731847612705'
2024-11-17T12:46:53,263 DEBUG [RS:0;0d751a622496:43029 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-17T12:46:53,263 DEBUG [RS:0;0d751a622496:43029 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-17T12:46:53,264 DEBUG [RS:0;0d751a622496:43029 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-17T12:46:53,264 INFO  [RS:0;0d751a622496:43029 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-17T12:46:53,264 INFO  [RS:0;0d751a622496:43029 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-17T12:46:53,366 INFO  [RS:0;0d751a622496:43029 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C43029%2C1731847612705, suffix=, logDir=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705, archiveDir=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/oldWALs, maxLogs=32
2024-11-17T12:46:53,367 INFO  [RS:0;0d751a622496:43029 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C43029%2C1731847612705.1731847613367
2024-11-17T12:46:53,374 INFO  [RS:0;0d751a622496:43029 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847613367
2024-11-17T12:46:53,397 DEBUG [RS:0;0d751a622496:43029 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34235:34235),(127.0.0.1/127.0.0.1:38513:38513)]
2024-11-17T12:46:53,508 DEBUG [0d751a622496:37969 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-17T12:46:53,508 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0d751a622496,43029,1731847612705
2024-11-17T12:46:53,510 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0d751a622496,43029,1731847612705, state=OPENING
2024-11-17T12:46:53,548 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-17T12:46:53,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43029-0x1014a1efb3c0001, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:53,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:46:53,557 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T12:46:53,557 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:46:53,557 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:46:53,557 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0d751a622496,43029,1731847612705}]
2024-11-17T12:46:53,712 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-17T12:46:53,717 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50303, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-17T12:46:53,725 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-17T12:46:53,725 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T12:46:53,727 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C43029%2C1731847612705.meta, suffix=.meta, logDir=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705, archiveDir=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/oldWALs, maxLogs=32
2024-11-17T12:46:53,728 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
2024-11-17T12:46:53,735 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
2024-11-17T12:46:53,743 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38513:38513),(127.0.0.1/127.0.0.1:34235:34235)]
2024-11-17T12:46:53,747 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-17T12:46:53,747 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-17T12:46:53,747 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-17T12:46:53,747 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-17T12:46:53,747 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-17T12:46:53,747 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:46:53,747 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-17T12:46:53,747 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-17T12:46:53,750 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T12:46:53,751 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T12:46:53,751 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:53,751 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:46:53,751 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T12:46:53,752 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T12:46:53,752 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:53,753 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:46:53,753 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T12:46:53,753 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T12:46:53,754 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:53,754 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:46:53,754 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T12:46:53,755 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T12:46:53,755 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:53,756 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:46:53,756 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T12:46:53,757 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/hbase/meta/1588230740
2024-11-17T12:46:53,758 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/hbase/meta/1588230740
2024-11-17T12:46:53,761 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T12:46:53,761 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T12:46:53,762 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T12:46:53,763 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T12:46:53,764 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=712450, jitterRate=-0.09407314658164978}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T12:46:53,764 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-17T12:46:53,765 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731847613748Writing region info on filesystem at 1731847613748Initializing all the Stores at 1731847613749 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847613749Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847613750 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847613750Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847613750Cleaning up temporary data from old regions at 1731847613761 (+11 ms)Running coprocessor post-open hooks at 1731847613765 (+4 ms)Region opened successfully at 1731847613765
2024-11-17T12:46:53,766 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731847613712
2024-11-17T12:46:53,769 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-17T12:46:53,769 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-17T12:46:53,770 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0d751a622496,43029,1731847612705
2024-11-17T12:46:53,771 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0d751a622496,43029,1731847612705, state=OPEN
2024-11-17T12:46:53,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T12:46:53,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43029-0x1014a1efb3c0001, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T12:46:53,811 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0d751a622496,43029,1731847612705
2024-11-17T12:46:53,811 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:46:53,811 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:46:53,817 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-17T12:46:53,817 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0d751a622496,43029,1731847612705 in 254 msec
2024-11-17T12:46:53,821 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-17T12:46:53,821 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 713 msec
2024-11-17T12:46:53,822 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T12:46:53,822 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-17T12:46:53,823 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T12:46:53,823 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0d751a622496,43029,1731847612705, seqNum=-1]
2024-11-17T12:46:53,824 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T12:46:53,825 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36379, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T12:46:53,831 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 781 msec
2024-11-17T12:46:53,831 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731847613831, completionTime=-1
2024-11-17T12:46:53,832 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-17T12:46:53,832 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-17T12:46:53,834 INFO  [master/0d751a622496:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-17T12:46:53,834 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731847673834
2024-11-17T12:46:53,834 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731847733834
2024-11-17T12:46:53,834 INFO  [master/0d751a622496:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-17T12:46:53,834 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,37969,1731847612555-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:53,834 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,37969,1731847612555-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:53,834 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,37969,1731847612555-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:53,835 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0d751a622496:37969, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:53,835 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:53,835 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:53,837 DEBUG [master/0d751a622496:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-17T12:46:53,839 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.066sec
2024-11-17T12:46:53,839 INFO  [master/0d751a622496:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-17T12:46:53,839 INFO  [master/0d751a622496:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-17T12:46:53,839 INFO  [master/0d751a622496:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-17T12:46:53,839 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-17T12:46:53,839 INFO  [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-17T12:46:53,839 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,37969,1731847612555-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T12:46:53,840 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,37969,1731847612555-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-17T12:46:53,842 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-17T12:46:53,842 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-17T12:46:53,842 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,37969,1731847612555-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:53,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe657aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T12:46:53,936 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0d751a622496,37969,-1 for getting cluster id
2024-11-17T12:46:53,936 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-17T12:46:53,939 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'eb423186-dfd2-460f-bbe3-1e398e27b24d'
2024-11-17T12:46:53,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-17T12:46:53,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "eb423186-dfd2-460f-bbe3-1e398e27b24d"

2024-11-17T12:46:53,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f633512, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T12:46:53,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0d751a622496,37969,-1]
2024-11-17T12:46:53,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-17T12:46:53,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:46:53,944 INFO  [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58514, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-17T12:46:53,945 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337c5dd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T12:46:53,946 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T12:46:53,947 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0d751a622496,43029,1731847612705, seqNum=-1]
2024-11-17T12:46:53,948 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T12:46:53,950 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T12:46:53,952 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0d751a622496,37969,1731847612555
2024-11-17T12:46:53,953 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:46:53,957 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-17T12:46:53,976 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/0d751a622496:0 server-side Connection retries=45
2024-11-17T12:46:53,976 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:46:53,976 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T12:46:53,976 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T12:46:53,977 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:46:53,977 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T12:46:53,977 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-17T12:46:53,977 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T12:46:53,978 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37689
2024-11-17T12:46:53,979 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37689 connecting to ZooKeeper ensemble=127.0.0.1:62721
2024-11-17T12:46:53,980 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:46:53,981 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:46:54,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:376890x0, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T12:46:54,015 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37689-0x1014a1efb3c0002 connected
2024-11-17T12:46:54,015 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:37689-0x1014a1efb3c0002, quorum=127.0.0.1:62721, baseZNode=/hbase Set watcher on existing znode=/hbase/running
2024-11-17T12:46:54,015 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT
2024-11-17T12:46:54,016 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-17T12:46:54,017 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-17T12:46:54,018 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:37689-0x1014a1efb3c0002, quorum=127.0.0.1:62721, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-17T12:46:54,020 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37689-0x1014a1efb3c0002, quorum=127.0.0.1:62721, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T12:46:54,021 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37689
2024-11-17T12:46:54,022 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37689
2024-11-17T12:46:54,024 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37689
2024-11-17T12:46:54,025 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37689
2024-11-17T12:46:54,025 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37689
2024-11-17T12:46:54,028 INFO  [RS:1;0d751a622496:37689 {}] regionserver.HRegionServer(746): ClusterId : eb423186-dfd2-460f-bbe3-1e398e27b24d
2024-11-17T12:46:54,028 DEBUG [RS:1;0d751a622496:37689 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-17T12:46:54,040 DEBUG [RS:1;0d751a622496:37689 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-17T12:46:54,040 DEBUG [RS:1;0d751a622496:37689 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-17T12:46:54,049 DEBUG [RS:1;0d751a622496:37689 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-17T12:46:54,049 DEBUG [RS:1;0d751a622496:37689 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d444263, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0d751a622496/172.17.0.2:0
2024-11-17T12:46:54,063 DEBUG [RS:1;0d751a622496:37689 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;0d751a622496:37689
2024-11-17T12:46:54,063 INFO  [RS:1;0d751a622496:37689 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-17T12:46:54,063 INFO  [RS:1;0d751a622496:37689 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-17T12:46:54,063 DEBUG [RS:1;0d751a622496:37689 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-17T12:46:54,064 INFO  [RS:1;0d751a622496:37689 {}] regionserver.HRegionServer(2659): reportForDuty to master=0d751a622496,37969,1731847612555 with port=37689, startcode=1731847613976
2024-11-17T12:46:54,064 DEBUG [RS:1;0d751a622496:37689 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-17T12:46:54,066 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43137, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-11-17T12:46:54,066 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37969 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0d751a622496,37689,1731847613976
2024-11-17T12:46:54,067 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37969 {}] master.ServerManager(517): Registering regionserver=0d751a622496,37689,1731847613976
2024-11-17T12:46:54,069 DEBUG [RS:1;0d751a622496:37689 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8
2024-11-17T12:46:54,069 DEBUG [RS:1;0d751a622496:37689 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42079
2024-11-17T12:46:54,069 DEBUG [RS:1;0d751a622496:37689 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-17T12:46:54,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T12:46:54,078 DEBUG [RS:1;0d751a622496:37689 {}] zookeeper.ZKUtil(111): regionserver:37689-0x1014a1efb3c0002, quorum=127.0.0.1:62721, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0d751a622496,37689,1731847613976
2024-11-17T12:46:54,079 WARN  [RS:1;0d751a622496:37689 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T12:46:54,079 INFO  [RS:1;0d751a622496:37689 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T12:46:54,079 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0d751a622496,37689,1731847613976]
2024-11-17T12:46:54,079 DEBUG [RS:1;0d751a622496:37689 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976
2024-11-17T12:46:54,083 INFO  [RS:1;0d751a622496:37689 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-17T12:46:54,086 INFO  [RS:1;0d751a622496:37689 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-17T12:46:54,086 INFO  [RS:1;0d751a622496:37689 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-17T12:46:54,086 INFO  [RS:1;0d751a622496:37689 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:54,086 INFO  [RS:1;0d751a622496:37689 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-17T12:46:54,087 INFO  [RS:1;0d751a622496:37689 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-17T12:46:54,087 INFO  [RS:1;0d751a622496:37689 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:54,087 DEBUG [RS:1;0d751a622496:37689 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:54,088 DEBUG [RS:1;0d751a622496:37689 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:54,088 DEBUG [RS:1;0d751a622496:37689 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:54,088 DEBUG [RS:1;0d751a622496:37689 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:54,088 DEBUG [RS:1;0d751a622496:37689 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:54,088 DEBUG [RS:1;0d751a622496:37689 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0, corePoolSize=2, maxPoolSize=2
2024-11-17T12:46:54,088 DEBUG [RS:1;0d751a622496:37689 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:54,088 DEBUG [RS:1;0d751a622496:37689 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:54,088 DEBUG [RS:1;0d751a622496:37689 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:54,088 DEBUG [RS:1;0d751a622496:37689 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:54,088 DEBUG [RS:1;0d751a622496:37689 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:54,088 DEBUG [RS:1;0d751a622496:37689 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:46:54,088 DEBUG [RS:1;0d751a622496:37689 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0d751a622496:0, corePoolSize=3, maxPoolSize=3
2024-11-17T12:46:54,088 DEBUG [RS:1;0d751a622496:37689 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0, corePoolSize=3, maxPoolSize=3
2024-11-17T12:46:54,089 INFO  [RS:1;0d751a622496:37689 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:54,089 INFO  [RS:1;0d751a622496:37689 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:54,089 INFO  [RS:1;0d751a622496:37689 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:54,089 INFO  [RS:1;0d751a622496:37689 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:54,089 INFO  [RS:1;0d751a622496:37689 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:54,089 INFO  [RS:1;0d751a622496:37689 {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,37689,1731847613976-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T12:46:54,103 INFO  [RS:1;0d751a622496:37689 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-17T12:46:54,104 INFO  [RS:1;0d751a622496:37689 {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,37689,1731847613976-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:54,104 INFO  [RS:1;0d751a622496:37689 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:54,104 INFO  [RS:1;0d751a622496:37689 {}] regionserver.Replication(171): 0d751a622496,37689,1731847613976 started
2024-11-17T12:46:54,117 INFO  [RS:1;0d751a622496:37689 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:46:54,117 INFO  [RS:1;0d751a622496:37689 {}] regionserver.HRegionServer(1482): Serving as 0d751a622496,37689,1731847613976, RpcServer on 0d751a622496/172.17.0.2:37689, sessionid=0x1014a1efb3c0002
2024-11-17T12:46:54,117 DEBUG [RS:1;0d751a622496:37689 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-17T12:46:54,117 DEBUG [RS:1;0d751a622496:37689 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0d751a622496,37689,1731847613976
2024-11-17T12:46:54,117 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;0d751a622496:37689,5,FailOnTimeoutGroup]
2024-11-17T12:46:54,117 DEBUG [RS:1;0d751a622496:37689 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0d751a622496,37689,1731847613976'
2024-11-17T12:46:54,117 DEBUG [RS:1;0d751a622496:37689 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-17T12:46:54,117 INFO  [Time-limited test {}] wal.TestLogRolling(207): Replication=2
2024-11-17T12:46:54,118 DEBUG [RS:1;0d751a622496:37689 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-17T12:46:54,118 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-17T12:46:54,118 DEBUG [RS:1;0d751a622496:37689 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-17T12:46:54,118 DEBUG [RS:1;0d751a622496:37689 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-17T12:46:54,118 DEBUG [RS:1;0d751a622496:37689 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0d751a622496,37689,1731847613976
2024-11-17T12:46:54,118 DEBUG [RS:1;0d751a622496:37689 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0d751a622496,37689,1731847613976'
2024-11-17T12:46:54,118 DEBUG [RS:1;0d751a622496:37689 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-17T12:46:54,119 DEBUG [RS:1;0d751a622496:37689 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-17T12:46:54,119 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is 0d751a622496,37969,1731847612555
2024-11-17T12:46:54,119 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@60c9ef5b
2024-11-17T12:46:54,119 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-17T12:46:54,119 DEBUG [RS:1;0d751a622496:37689 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-17T12:46:54,119 INFO  [RS:1;0d751a622496:37689 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-17T12:46:54,119 INFO  [RS:1;0d751a622496:37689 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-17T12:46:54,121 INFO  [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-17T12:46:54,121 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37969 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-17T12:46:54,122 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37969 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-17T12:46:54,122 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37969 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-17T12:46:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37969 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath
2024-11-17T12:46:54,125 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION
2024-11-17T12:46:54,125 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:54,125 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37969 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnDatanodeDeath"
 procId is: 4
2024-11-17T12:46:54,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-17T12:46:54,126 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-17T12:46:54,134 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33741 is added to blk_1073741835_1011 (size=393)
2024-11-17T12:46:54,134 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741835_1011 (size=393)
2024-11-17T12:46:54,136 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dc2b8c9bdc7b25edc69ab039c0619f99, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8
2024-11-17T12:46:54,142 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33741 is added to blk_1073741836_1012 (size=76)
2024-11-17T12:46:54,142 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45021 is added to blk_1073741836_1012 (size=76)
2024-11-17T12:46:54,143 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:46:54,143 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing dc2b8c9bdc7b25edc69ab039c0619f99, disabling compactions & flushes
2024-11-17T12:46:54,143 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.
2024-11-17T12:46:54,143 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.
2024-11-17T12:46:54,143 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99. after waiting 0 ms
2024-11-17T12:46:54,143 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.
2024-11-17T12:46:54,143 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.
2024-11-17T12:46:54,143 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for dc2b8c9bdc7b25edc69ab039c0619f99:
Waiting for close lock at 1731847614143Disabling compacts and flushes for region at 1731847614143Disabling writes for close at 1731847614143Writing region close event to WAL at 1731847614143Closed at 1731847614143
2024-11-17T12:46:54,145 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META
2024-11-17T12:46:54,145 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731847614145"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731847614145"}]},"ts":"1731847614145"}
2024-11-17T12:46:54,147 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-17T12:46:54,149 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-17T12:46:54,149 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731847614149"}]},"ts":"1731847614149"}
2024-11-17T12:46:54,151 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta
2024-11-17T12:46:54,151 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=dc2b8c9bdc7b25edc69ab039c0619f99, ASSIGN}]
2024-11-17T12:46:54,152 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=dc2b8c9bdc7b25edc69ab039c0619f99, ASSIGN
2024-11-17T12:46:54,153 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=dc2b8c9bdc7b25edc69ab039c0619f99, ASSIGN; state=OFFLINE, location=0d751a622496,43029,1731847612705; forceNewPlan=false, retain=false
2024-11-17T12:46:54,223 INFO  [RS:1;0d751a622496:37689 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C37689%2C1731847613976, suffix=, logDir=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976, archiveDir=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/oldWALs, maxLogs=32
2024-11-17T12:46:54,225 INFO  [RS:1;0d751a622496:37689 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C37689%2C1731847613976.1731847614224
2024-11-17T12:46:54,235 INFO  [RS:1;0d751a622496:37689 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
2024-11-17T12:46:54,238 DEBUG [RS:1;0d751a622496:37689 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34235:34235),(127.0.0.1/127.0.0.1:38513:38513)]
2024-11-17T12:46:54,304 INFO  [0d751a622496:37969 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-11-17T12:46:54,305 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=dc2b8c9bdc7b25edc69ab039c0619f99, regionState=OPENING, regionLocation=0d751a622496,43029,1731847612705
2024-11-17T12:46:54,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=dc2b8c9bdc7b25edc69ab039c0619f99, ASSIGN because future has completed
2024-11-17T12:46:54,310 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc2b8c9bdc7b25edc69ab039c0619f99, server=0d751a622496,43029,1731847612705}]
2024-11-17T12:46:54,477 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.
2024-11-17T12:46:54,477 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => dc2b8c9bdc7b25edc69ab039c0619f99, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.', STARTKEY => '', ENDKEY => ''}
2024-11-17T12:46:54,478 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath dc2b8c9bdc7b25edc69ab039c0619f99
2024-11-17T12:46:54,478 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:46:54,478 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for dc2b8c9bdc7b25edc69ab039c0619f99
2024-11-17T12:46:54,478 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for dc2b8c9bdc7b25edc69ab039c0619f99
2024-11-17T12:46:54,480 INFO  [StoreOpener-dc2b8c9bdc7b25edc69ab039c0619f99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region dc2b8c9bdc7b25edc69ab039c0619f99 
2024-11-17T12:46:54,482 INFO  [StoreOpener-dc2b8c9bdc7b25edc69ab039c0619f99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc2b8c9bdc7b25edc69ab039c0619f99 columnFamilyName info
2024-11-17T12:46:54,482 DEBUG [StoreOpener-dc2b8c9bdc7b25edc69ab039c0619f99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:46:54,482 INFO  [StoreOpener-dc2b8c9bdc7b25edc69ab039c0619f99-1 {}] regionserver.HStore(327): Store=dc2b8c9bdc7b25edc69ab039c0619f99/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:46:54,483 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for dc2b8c9bdc7b25edc69ab039c0619f99
2024-11-17T12:46:54,484 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99
2024-11-17T12:46:54,484 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99
2024-11-17T12:46:54,485 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for dc2b8c9bdc7b25edc69ab039c0619f99
2024-11-17T12:46:54,485 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for dc2b8c9bdc7b25edc69ab039c0619f99
2024-11-17T12:46:54,487 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for dc2b8c9bdc7b25edc69ab039c0619f99
2024-11-17T12:46:54,489 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T12:46:54,490 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened dc2b8c9bdc7b25edc69ab039c0619f99; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=789542, jitterRate=0.003954708576202393}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-17T12:46:54,490 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dc2b8c9bdc7b25edc69ab039c0619f99
2024-11-17T12:46:54,491 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for dc2b8c9bdc7b25edc69ab039c0619f99:
Running coprocessor pre-open hook at 1731847614478Writing region info on filesystem at 1731847614478Initializing all the Stores at 1731847614479 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847614479Cleaning up temporary data from old regions at 1731847614485 (+6 ms)Running coprocessor post-open hooks at 1731847614490 (+5 ms)Region opened successfully at 1731847614490
2024-11-17T12:46:54,492 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99., pid=6, masterSystemTime=1731847614467
2024-11-17T12:46:54,494 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.
2024-11-17T12:46:54,494 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.
2024-11-17T12:46:54,496 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=dc2b8c9bdc7b25edc69ab039c0619f99, regionState=OPEN, openSeqNum=2, regionLocation=0d751a622496,43029,1731847612705
2024-11-17T12:46:54,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc2b8c9bdc7b25edc69ab039c0619f99, server=0d751a622496,43029,1731847612705 because future has completed
2024-11-17T12:46:54,502 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-17T12:46:54,502 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure dc2b8c9bdc7b25edc69ab039c0619f99, server=0d751a622496,43029,1731847612705 in 190 msec
2024-11-17T12:46:54,505 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-17T12:46:54,505 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=dc2b8c9bdc7b25edc69ab039c0619f99, ASSIGN in 351 msec
2024-11-17T12:46:54,506 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-17T12:46:54,506 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731847614506"}]},"ts":"1731847614506"}
2024-11-17T12:46:54,509 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta
2024-11-17T12:46:54,510 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION
2024-11-17T12:46:54,512 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 388 msec
2024-11-17T12:46:59,254 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-17T12:46:59,256 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:46:59,276 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:46:59,278 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:46:59,278 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:46:59,285 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath'
2024-11-17T12:47:00,480 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-17T12:47:00,480 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-17T12:47:00,482 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-17T12:47:00,482 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer
2024-11-17T12:47:00,484 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T12:47:00,484 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-17T12:47:04,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37969 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-17T12:47:04,170 INFO  [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed
2024-11-17T12:47:04,171 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100
2024-11-17T12:47:04,175 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath
2024-11-17T12:47:04,175 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.
2024-11-17T12:47:04,190 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:47:04,193 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:47:04,194 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:47:04,194 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:47:04,194 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-17T12:47:04,194 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a9d9bc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:47:04,195 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3abd4c2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:47:04,288 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5e1eaefc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/java.io.tmpdir/jetty-localhost-42735-hadoop-hdfs-3_4_1-tests_jar-_-any-1360447532328855026/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:47:04,288 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@64b89ed5{HTTP/1.1, (http/1.1)}{localhost:42735}
2024-11-17T12:47:04,288 INFO  [Time-limited test {}] server.Server(415): Started @119530ms
2024-11-17T12:47:04,289 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T12:47:04,319 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:47:04,322 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:47:04,323 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:47:04,323 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:47:04,324 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T12:47:04,324 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@574823ec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:47:04,324 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@463ced82{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:47:04,418 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4b9d70b4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/java.io.tmpdir/jetty-localhost-39025-hadoop-hdfs-3_4_1-tests_jar-_-any-4496477088213552448/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:47:04,419 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@47432b7b{HTTP/1.1, (http/1.1)}{localhost:39025}
2024-11-17T12:47:04,419 INFO  [Time-limited test {}] server.Server(415): Started @119660ms
2024-11-17T12:47:04,420 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T12:47:04,452 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:47:04,455 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:47:04,458 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:47:04,458 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:47:04,458 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-17T12:47:04,458 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5af75f98{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:47:04,459 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28eb02f0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:47:04,559 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@360084d5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/java.io.tmpdir/jetty-localhost-38923-hadoop-hdfs-3_4_1-tests_jar-_-any-14864350533614434930/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:47:04,560 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@32cdb76b{HTTP/1.1, (http/1.1)}{localhost:38923}
2024-11-17T12:47:04,560 INFO  [Time-limited test {}] server.Server(415): Started @119801ms
2024-11-17T12:47:04,561 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T12:47:05,234 WARN  [Thread-867 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data6/current/BP-1311131906-172.17.0.2-1731847610939/current, will proceed with Du for space computation calculation, 
2024-11-17T12:47:05,234 WARN  [Thread-866 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data5/current/BP-1311131906-172.17.0.2-1731847610939/current, will proceed with Du for space computation calculation, 
2024-11-17T12:47:05,254 WARN  [Thread-808 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T12:47:05,257 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa89540403fe6104f with lease ID 0xfc001fc3e00951b: Processing first storage report for DS-161e2e36-93ec-4012-b9bc-7865bcb4b452 from datanode DatanodeRegistration(127.0.0.1:37111, datanodeUuid=ef486935-aa1e-4fc3-adea-18d6181aad55, infoPort=39527, infoSecurePort=0, ipcPort=37001, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939)
2024-11-17T12:47:05,257 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa89540403fe6104f with lease ID 0xfc001fc3e00951b: from storage DS-161e2e36-93ec-4012-b9bc-7865bcb4b452 node DatanodeRegistration(127.0.0.1:37111, datanodeUuid=ef486935-aa1e-4fc3-adea-18d6181aad55, infoPort=39527, infoSecurePort=0, ipcPort=37001, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:47:05,257 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa89540403fe6104f with lease ID 0xfc001fc3e00951b: Processing first storage report for DS-bbdec07e-21dc-4ff6-baa9-9d680824135b from datanode DatanodeRegistration(127.0.0.1:37111, datanodeUuid=ef486935-aa1e-4fc3-adea-18d6181aad55, infoPort=39527, infoSecurePort=0, ipcPort=37001, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939)
2024-11-17T12:47:05,257 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa89540403fe6104f with lease ID 0xfc001fc3e00951b: from storage DS-bbdec07e-21dc-4ff6-baa9-9d680824135b node DatanodeRegistration(127.0.0.1:37111, datanodeUuid=ef486935-aa1e-4fc3-adea-18d6181aad55, infoPort=39527, infoSecurePort=0, ipcPort=37001, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:47:05,409 WARN  [Thread-878 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data7/current/BP-1311131906-172.17.0.2-1731847610939/current, will proceed with Du for space computation calculation, 
2024-11-17T12:47:05,409 WARN  [Thread-879 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data8/current/BP-1311131906-172.17.0.2-1731847610939/current, will proceed with Du for space computation calculation, 
2024-11-17T12:47:05,428 WARN  [Thread-830 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T12:47:05,431 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4ac8ab7e6146ad7d with lease ID 0xfc001fc3e00951c: Processing first storage report for DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b from datanode DatanodeRegistration(127.0.0.1:39533, datanodeUuid=6bb64b52-41b8-49af-bf13-ba9bcad2229b, infoPort=40685, infoSecurePort=0, ipcPort=41737, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939)
2024-11-17T12:47:05,431 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4ac8ab7e6146ad7d with lease ID 0xfc001fc3e00951c: from storage DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b node DatanodeRegistration(127.0.0.1:39533, datanodeUuid=6bb64b52-41b8-49af-bf13-ba9bcad2229b, infoPort=40685, infoSecurePort=0, ipcPort=41737, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:47:05,431 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4ac8ab7e6146ad7d with lease ID 0xfc001fc3e00951c: Processing first storage report for DS-14010b18-0532-4491-b575-30a4b81a17ae from datanode DatanodeRegistration(127.0.0.1:39533, datanodeUuid=6bb64b52-41b8-49af-bf13-ba9bcad2229b, infoPort=40685, infoSecurePort=0, ipcPort=41737, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939)
2024-11-17T12:47:05,431 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4ac8ab7e6146ad7d with lease ID 0xfc001fc3e00951c: from storage DS-14010b18-0532-4491-b575-30a4b81a17ae node DatanodeRegistration(127.0.0.1:39533, datanodeUuid=6bb64b52-41b8-49af-bf13-ba9bcad2229b, infoPort=40685, infoSecurePort=0, ipcPort=41737, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:47:05,541 WARN  [Thread-889 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data9/current/BP-1311131906-172.17.0.2-1731847610939/current, will proceed with Du for space computation calculation, 
2024-11-17T12:47:05,541 WARN  [Thread-890 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data10/current/BP-1311131906-172.17.0.2-1731847610939/current, will proceed with Du for space computation calculation, 
2024-11-17T12:47:05,562 WARN  [Thread-852 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T12:47:05,565 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x876ac74e97219e9b with lease ID 0xfc001fc3e00951d: Processing first storage report for DS-a1073542-bba2-4b3f-8007-c55f080d1c21 from datanode DatanodeRegistration(127.0.0.1:36351, datanodeUuid=bb9d3f68-6419-4917-a14a-560a6e9c2984, infoPort=42495, infoSecurePort=0, ipcPort=43779, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939)
2024-11-17T12:47:05,565 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x876ac74e97219e9b with lease ID 0xfc001fc3e00951d: from storage DS-a1073542-bba2-4b3f-8007-c55f080d1c21 node DatanodeRegistration(127.0.0.1:36351, datanodeUuid=bb9d3f68-6419-4917-a14a-560a6e9c2984, infoPort=42495, infoSecurePort=0, ipcPort=43779, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:47:05,565 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x876ac74e97219e9b with lease ID 0xfc001fc3e00951d: Processing first storage report for DS-6f9aea2b-4250-43a9-af37-dc466098d9a7 from datanode DatanodeRegistration(127.0.0.1:36351, datanodeUuid=bb9d3f68-6419-4917-a14a-560a6e9c2984, infoPort=42495, infoSecurePort=0, ipcPort=43779, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939)
2024-11-17T12:47:05,565 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x876ac74e97219e9b with lease ID 0xfc001fc3e00951d: from storage DS-6f9aea2b-4250-43a9-af37-dc466098d9a7 node DatanodeRegistration(127.0.0.1:36351, datanodeUuid=bb9d3f68-6419-4917-a14a-560a6e9c2984, infoPort=42495, infoSecurePort=0, ipcPort=43779, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:47:05,589 WARN  [ResponseProcessor for block BP-1311131906-172.17.0.2-1731847610939:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1311131906-172.17.0.2-1731847610939:blk_1073741833_1009
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:05,589 WARN  [ResponseProcessor for block BP-1311131906-172.17.0.2-1731847610939:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1311131906-172.17.0.2-1731847610939:blk_1073741830_1006
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:05,589 WARN  [ResponseProcessor for block BP-1311131906-172.17.0.2-1731847610939:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1311131906-172.17.0.2-1731847610939:blk_1073741837_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:05,589 WARN  [ResponseProcessor for block BP-1311131906-172.17.0.2-1731847610939:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1311131906-172.17.0.2-1731847610939:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-1311131906-172.17.0.2-1731847610939:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:05,590 WARN  [DataStreamer for file /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847613367 block BP-1311131906-172.17.0.2-1731847610939:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK], DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]) is bad.
2024-11-17T12:47:05,590 WARN  [DataStreamer for file /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224 block BP-1311131906-172.17.0.2-1731847610939:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK], DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]) is bad.
2024-11-17T12:47:05,590 WARN  [DataStreamer for file /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta block BP-1311131906-172.17.0.2-1731847610939:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK], DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]) is bad.
2024-11-17T12:47:05,590 WARN  [DataStreamer for file /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/WALs/0d751a622496,37969,1731847612555/0d751a622496%2C37969%2C1731847612555.1731847612841 block BP-1311131906-172.17.0.2-1731847610939:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK], DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]) is bad.
2024-11-17T12:47:05,590 WARN  [PacketResponder: BP-1311131906-172.17.0.2-1731847610939:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:33741] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:05,592 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:33338 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:45021:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33338 dst: /127.0.0.1:45021
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:05,592 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_475526744_22 at /127.0.0.1:33312 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:45021:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33312 dst: /127.0.0.1:45021
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:05,592 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1015699961_22 at /127.0.0.1:56018 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:33741:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56018 dst: /127.0.0.1:33741
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:05,592 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1015699961_22 at /127.0.0.1:33362 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:45021:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33362 dst: /127.0.0.1:45021
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:05,592 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:55974 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:33741:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:55974 dst: /127.0.0.1:33741
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:05,592 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_475526744_22 at /127.0.0.1:55938 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:33741:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:55938 dst: /127.0.0.1:33741
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:05,592 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:55984 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:33741:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:55984 dst: /127.0.0.1:33741
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:05,594 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:33330 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:45021:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33330 dst: /127.0.0.1:45021
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:05,595 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2ade06e3{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:47:05,595 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1baf7059{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:47:05,596 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:47:05,596 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@73b17c7e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:47:05,596 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@10a4d310{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/hadoop.log.dir/,STOPPED}
2024-11-17T12:47:05,597 WARN  [BP-1311131906-172.17.0.2-1731847610939 heartbeating to localhost/127.0.0.1:42079 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T12:47:05,597 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T12:47:05,598 WARN  [BP-1311131906-172.17.0.2-1731847610939 heartbeating to localhost/127.0.0.1:42079 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1311131906-172.17.0.2-1731847610939 (Datanode Uuid 368a0fe1-98fb-4e11-a7ad-3861a4d4f891) service to localhost/127.0.0.1:42079
2024-11-17T12:47:05,598 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T12:47:05,598 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data3/current/BP-1311131906-172.17.0.2-1731847610939 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:47:05,598 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data4/current/BP-1311131906-172.17.0.2-1731847610939 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:47:05,598 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T12:47:05,599 WARN  [DataStreamer for file /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224 block BP-1311131906-172.17.0.2-1731847610939:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:05,599 ERROR [org.apache.hadoop.hdfs.server.datanode.DataXceiver@4a4d12e5 {}] datanode.DataXceiver(331): 127.0.0.1:45021:DataXceiver error processing unknown operation  src: /127.0.0.1:58004 dst: /127.0.0.1:45021
java.io.IOException: Server closed.
	at org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.addPeer(DataXceiverServer.java:334) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:232) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:05,599 WARN  [DataStreamer for file /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta block BP-1311131906-172.17.0.2-1731847610939:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:05,599 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1015699961_22 at /127.0.0.1:58000 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:45021:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58000 dst: /127.0.0.1:45021
java.io.IOException: The stream is closed
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:117) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:914) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:05,599 WARN  [DataStreamer for file /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/WALs/0d751a622496,37969,1731847612555/0d751a622496%2C37969%2C1731847612555.1731847612841 block BP-1311131906-172.17.0.2-1731847610939:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:05,601 WARN  [DataStreamer for file /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847613367 block BP-1311131906-172.17.0.2-1731847610939:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:05,606 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@71ad97f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:47:05,606 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3f14f219{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:47:05,606 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:47:05,606 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7d4a4bdf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:47:05,607 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@48d478e7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/hadoop.log.dir/,STOPPED}
2024-11-17T12:47:05,608 WARN  [BP-1311131906-172.17.0.2-1731847610939 heartbeating to localhost/127.0.0.1:42079 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T12:47:05,608 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T12:47:05,608 WARN  [BP-1311131906-172.17.0.2-1731847610939 heartbeating to localhost/127.0.0.1:42079 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1311131906-172.17.0.2-1731847610939 (Datanode Uuid 158584b4-ac14-402d-849d-a116eafa11ed) service to localhost/127.0.0.1:42079
2024-11-17T12:47:05,608 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T12:47:05,608 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data1/current/BP-1311131906-172.17.0.2-1731847610939 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:47:05,608 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data2/current/BP-1311131906-172.17.0.2-1731847610939 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:47:05,609 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T12:47:05,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99., hostname=0d751a622496,43029,1731847612705, seqNum=2]
2024-11-17T12:47:05,613 ERROR [FSHLog-0-hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8-prefix:0d751a622496,43029,1731847612705 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:05,614 WARN  [FSHLog-0-hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8-prefix:0d751a622496,43029,1731847612705 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:05,614 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0d751a622496%2C43029%2C1731847612705:(num 1731847613367) roll requested
2024-11-17T12:47:05,614 INFO  [regionserver/0d751a622496:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C43029%2C1731847612705.1731847625614
2024-11-17T12:47:05,617 WARN  [Thread-902 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741838_1018
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:05,617 WARN  [Thread-902 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK], DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]) is bad.
2024-11-17T12:47:05,617 WARN  [Thread-902 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741838_1018
2024-11-17T12:47:05,620 WARN  [Thread-902 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]
2024-11-17T12:47:05,625 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:05,625 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:05,626 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:05,626 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:05,626 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:05,626 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847613367 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847625614
2024-11-17T12:47:05,626 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:05,626 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:05,627 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39527:39527),(127.0.0.1/127.0.0.1:42495:42495)]
2024-11-17T12:47:05,627 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847613367 is not closed yet, will try archiving it next time
2024-11-17T12:47:05,627 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils
2024-11-17T12:47:05,628 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease()
2024-11-17T12:47:05,628 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847613367
2024-11-17T12:47:05,631 WARN  [IPC Server handler 0 on default port 42079 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847613367 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741833_1009
2024-11-17T12:47:05,634 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847613367 after 4ms
2024-11-17T12:47:06,090 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:06,817 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:07,627 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:07,629 INFO  [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847625614
2024-11-17T12:47:07,630 WARN  [ResponseProcessor for block BP-1311131906-172.17.0.2-1731847610939:blk_1073741839_1019 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1311131906-172.17.0.2-1731847610939:blk_1073741839_1019
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:07,632 WARN  [DataStreamer for file /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847625614 block BP-1311131906-172.17.0.2-1731847610939:blk_1073741839_1019 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741839_1019 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK], DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]) is bad.
2024-11-17T12:47:07,632 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:59180 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741839_1019] {}] datanode.DataXceiver(331): 127.0.0.1:37111:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:59180 dst: /127.0.0.1:37111
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:07,633 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:44732 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741839_1019] {}] datanode.DataXceiver(331): 127.0.0.1:36351:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:44732 dst: /127.0.0.1:36351
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:07,648 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5e1eaefc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:47:07,649 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@64b89ed5{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:47:07,649 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:47:07,649 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3abd4c2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:47:07,649 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a9d9bc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/hadoop.log.dir/,STOPPED}
2024-11-17T12:47:07,650 WARN  [BP-1311131906-172.17.0.2-1731847610939 heartbeating to localhost/127.0.0.1:42079 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T12:47:07,650 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T12:47:07,650 WARN  [BP-1311131906-172.17.0.2-1731847610939 heartbeating to localhost/127.0.0.1:42079 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1311131906-172.17.0.2-1731847610939 (Datanode Uuid ef486935-aa1e-4fc3-adea-18d6181aad55) service to localhost/127.0.0.1:42079
2024-11-17T12:47:07,651 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T12:47:07,651 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data5/current/BP-1311131906-172.17.0.2-1731847610939 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:47:07,651 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data6/current/BP-1311131906-172.17.0.2-1731847610939 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:47:07,651 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T12:47:08,090 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:08,817 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:09,628 WARN  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]]
2024-11-17T12:47:09,628 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:09,628 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0d751a622496%2C43029%2C1731847612705:(num 1731847625614) roll requested
2024-11-17T12:47:09,629 INFO  [regionserver/0d751a622496:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C43029%2C1731847612705.1731847629629
2024-11-17T12:47:09,633 WARN  [Thread-911 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741840_1022
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:09,633 WARN  [Thread-911 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741840_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK], DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]) is bad.
2024-11-17T12:47:09,634 WARN  [Thread-911 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741840_1022
2024-11-17T12:47:09,634 WARN  [Thread-911 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]
2024-11-17T12:47:09,635 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847613367 after 4007ms
2024-11-17T12:47:09,638 WARN  [Thread-911 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1023
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33741
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:09,638 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:44752 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741841_1023] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data10]'}, localName='127.0.0.1:36351', datanodeUuid='bb9d3f68-6419-4917-a14a-560a6e9c2984', xmitsInProgress=0}:Exception transferring block BP-1311131906-172.17.0.2-1731847610939:blk_1073741841_1023 to mirror 127.0.0.1:33741
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:09,638 WARN  [Thread-911 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741841_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK], DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]) is bad.
2024-11-17T12:47:09,638 WARN  [Thread-911 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741841_1023
2024-11-17T12:47:09,638 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:44752 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741841_1023] {}] datanode.BlockReceiver(316): Block 1073741841 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-17T12:47:09,639 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:44752 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741841_1023] {}] datanode.DataXceiver(331): 127.0.0.1:36351:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:44752 dst: /127.0.0.1:36351
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:09,639 WARN  [Thread-911 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]
2024-11-17T12:47:09,641 WARN  [Thread-911 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1024
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:09,641 WARN  [Thread-911 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741842_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK], DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]) is bad.
2024-11-17T12:47:09,641 WARN  [Thread-911 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741842_1024
2024-11-17T12:47:09,641 WARN  [Thread-911 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]
2024-11-17T12:47:09,646 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:09,646 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:09,646 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:09,646 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:09,646 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:09,647 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847625614 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847629629
2024-11-17T12:47:09,648 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40685:40685),(127.0.0.1/127.0.0.1:42495:42495)]
2024-11-17T12:47:09,648 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847613367 is not closed yet, will try archiving it next time
2024-11-17T12:47:09,648 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847625614 is not closed yet, will try archiving it next time
2024-11-17T12:47:09,649 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36351 is added to blk_1073741839_1021 (size=2431)
2024-11-17T12:47:09,656 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-17T12:47:10,050 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847613367 is not closed yet, will try archiving it next time
2024-11-17T12:47:10,091 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:10,818 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:11,648 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:11,661 WARN  [ResponseProcessor for block BP-1311131906-172.17.0.2-1731847610939:blk_1073741843_1025 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1311131906-172.17.0.2-1731847610939:blk_1073741843_1025
java.io.IOException: Bad response ERROR for BP-1311131906-172.17.0.2-1731847610939:blk_1073741843_1025 from datanode DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:11,662 WARN  [DataStreamer for file /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847629629 block BP-1311131906-172.17.0.2-1731847610939:blk_1073741843_1025 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741843_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39533,DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b,DISK], DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]) is bad.
2024-11-17T12:47:11,662 WARN  [PacketResponder: BP-1311131906-172.17.0.2-1731847610939:blk_1073741843_1025, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:36351] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:11,664 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:35562 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741843_1025] {}] datanode.DataXceiver(331): 127.0.0.1:39533:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35562 dst: /127.0.0.1:39533
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:11,664 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:44768 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741843_1025] {}] datanode.DataXceiver(331): 127.0.0.1:36351:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:44768 dst: /127.0.0.1:36351
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:11,698 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@360084d5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:47:11,699 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@32cdb76b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:47:11,699 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:47:11,699 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28eb02f0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:47:11,699 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5af75f98{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/hadoop.log.dir/,STOPPED}
2024-11-17T12:47:11,700 WARN  [BP-1311131906-172.17.0.2-1731847610939 heartbeating to localhost/127.0.0.1:42079 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T12:47:11,700 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T12:47:11,700 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T12:47:11,700 WARN  [BP-1311131906-172.17.0.2-1731847610939 heartbeating to localhost/127.0.0.1:42079 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1311131906-172.17.0.2-1731847610939 (Datanode Uuid bb9d3f68-6419-4917-a14a-560a6e9c2984) service to localhost/127.0.0.1:42079
2024-11-17T12:47:11,701 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data9/current/BP-1311131906-172.17.0.2-1731847610939 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:47:11,701 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data10/current/BP-1311131906-172.17.0.2-1731847610939 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:47:11,702 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T12:47:11,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43029 {}] regionserver.HRegion(8855): Flush requested on dc2b8c9bdc7b25edc69ab039c0619f99
2024-11-17T12:47:11,715 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc2b8c9bdc7b25edc69ab039c0619f99 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-17T12:47:11,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/.tmp/info/b492cab2e0a74b6ab93a360f50888d77 is 1080, key is row0002/info:/1731847627653/Put/seqid=0
2024-11-17T12:47:11,737 WARN  [Thread-921 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37111
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:11,737 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:35578 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741844_1027] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data8]'}, localName='127.0.0.1:39533', datanodeUuid='6bb64b52-41b8-49af-bf13-ba9bcad2229b', xmitsInProgress=0}:Exception transferring block BP-1311131906-172.17.0.2-1731847610939:blk_1073741844_1027 to mirror 127.0.0.1:37111
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:11,737 WARN  [Thread-921 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39533,DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b,DISK], DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]) is bad.
2024-11-17T12:47:11,737 WARN  [Thread-921 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741844_1027
2024-11-17T12:47:11,737 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:35578 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741844_1027] {}] datanode.BlockReceiver(316): Block 1073741844 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-17T12:47:11,737 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:35578 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741844_1027] {}] datanode.DataXceiver(331): 127.0.0.1:39533:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35578 dst: /127.0.0.1:39533
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:11,738 WARN  [Thread-921 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]
2024-11-17T12:47:11,739 WARN  [Thread-921 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:11,740 WARN  [Thread-921 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK], DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]) is bad.
2024-11-17T12:47:11,740 WARN  [Thread-921 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741845_1028
2024-11-17T12:47:11,741 WARN  [Thread-921 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]
2024-11-17T12:47:11,742 WARN  [Thread-921 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:11,742 WARN  [Thread-921 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK], DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]) is bad.
2024-11-17T12:47:11,742 WARN  [Thread-921 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741846_1029
2024-11-17T12:47:11,743 WARN  [Thread-921 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]
2024-11-17T12:47:11,744 WARN  [Thread-921 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:11,744 WARN  [Thread-921 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK], DatanodeInfoWithStorage[127.0.0.1:39533,DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]) is bad.
2024-11-17T12:47:11,744 WARN  [Thread-921 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741847_1030
2024-11-17T12:47:11,745 WARN  [Thread-921 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]
2024-11-17T12:47:11,746 WARN  [IPC Server handler 3 on default port 42079 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-17T12:47:11,746 WARN  [IPC Server handler 3 on default port 42079 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-17T12:47:11,746 WARN  [IPC Server handler 3 on default port 42079 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-17T12:47:11,749 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741848_1031 (size=10347)
2024-11-17T12:47:12,091 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:12,150 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/.tmp/info/b492cab2e0a74b6ab93a360f50888d77
2024-11-17T12:47:12,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/.tmp/info/b492cab2e0a74b6ab93a360f50888d77 as hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/b492cab2e0a74b6ab93a360f50888d77
2024-11-17T12:47:12,168 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/b492cab2e0a74b6ab93a360f50888d77, entries=5, sequenceid=11, filesize=10.1 K
2024-11-17T12:47:12,169 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for dc2b8c9bdc7b25edc69ab039c0619f99 in 455ms, sequenceid=11, compaction requested=false
2024-11-17T12:47:12,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc2b8c9bdc7b25edc69ab039c0619f99:

2024-11-17T12:47:12,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43029 {}] regionserver.HRegion(8855): Flush requested on dc2b8c9bdc7b25edc69ab039c0619f99
2024-11-17T12:47:12,340 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc2b8c9bdc7b25edc69ab039c0619f99 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB
2024-11-17T12:47:12,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/.tmp/info/fc2a95a1cad1442ea020ad7e66fc262f is 1080, key is row0007/info:/1731847631716/Put/seqid=0
2024-11-17T12:47:12,346 WARN  [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:12,346 WARN  [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK], DatanodeInfoWithStorage[127.0.0.1:39533,DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]) is bad.
2024-11-17T12:47:12,346 WARN  [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741849_1032
2024-11-17T12:47:12,347 WARN  [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]
2024-11-17T12:47:12,348 WARN  [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:12,348 WARN  [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK], DatanodeInfoWithStorage[127.0.0.1:39533,DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]) is bad.
2024-11-17T12:47:12,348 WARN  [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741850_1033
2024-11-17T12:47:12,349 WARN  [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]
2024-11-17T12:47:12,350 WARN  [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:12,350 WARN  [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK], DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]) is bad.
2024-11-17T12:47:12,350 WARN  [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741851_1034
2024-11-17T12:47:12,351 WARN  [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]
2024-11-17T12:47:12,353 WARN  [Thread-927 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:45021
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:12,353 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:45488 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741852_1035] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data8]'}, localName='127.0.0.1:39533', datanodeUuid='6bb64b52-41b8-49af-bf13-ba9bcad2229b', xmitsInProgress=0}:Exception transferring block BP-1311131906-172.17.0.2-1731847610939:blk_1073741852_1035 to mirror 127.0.0.1:45021
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:12,353 WARN  [Thread-927 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39533,DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b,DISK], DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]) is bad.
2024-11-17T12:47:12,353 WARN  [Thread-927 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741852_1035
2024-11-17T12:47:12,353 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:45488 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741852_1035] {}] datanode.BlockReceiver(316): Block 1073741852 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-17T12:47:12,353 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:45488 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741852_1035] {}] datanode.DataXceiver(331): 127.0.0.1:39533:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45488 dst: /127.0.0.1:39533
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:12,354 WARN  [Thread-927 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]
2024-11-17T12:47:12,354 WARN  [IPC Server handler 3 on default port 42079 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-17T12:47:12,354 WARN  [IPC Server handler 3 on default port 42079 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-17T12:47:12,354 WARN  [IPC Server handler 3 on default port 42079 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-17T12:47:12,357 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741853_1036 (size=12506)
2024-11-17T12:47:12,759 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/.tmp/info/fc2a95a1cad1442ea020ad7e66fc262f
2024-11-17T12:47:12,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/.tmp/info/fc2a95a1cad1442ea020ad7e66fc262f as hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/fc2a95a1cad1442ea020ad7e66fc262f
2024-11-17T12:47:12,781 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/fc2a95a1cad1442ea020ad7e66fc262f, entries=7, sequenceid=24, filesize=12.2 K
2024-11-17T12:47:12,783 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2150 for dc2b8c9bdc7b25edc69ab039c0619f99 in 443ms, sequenceid=24, compaction requested=false
2024-11-17T12:47:12,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc2b8c9bdc7b25edc69ab039c0619f99:

2024-11-17T12:47:12,783 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K
2024-11-17T12:47:12,783 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:47:12,783 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/fc2a95a1cad1442ea020ad7e66fc262f because midkey is the same as first or last row
2024-11-17T12:47:12,819 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:13,441 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@2bea8b4b[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39533, datanodeUuid=6bb64b52-41b8-49af-bf13-ba9bcad2229b, infoPort=40685, infoSecurePort=0, ipcPort=41737, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939):Failed to transfer BP-1311131906-172.17.0.2-1731847610939:blk_1073741848_1031 to 127.0.0.1:37111 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:13,441 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4c81f1c7[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39533, datanodeUuid=6bb64b52-41b8-49af-bf13-ba9bcad2229b, infoPort=40685, infoSecurePort=0, ipcPort=41737, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939):Failed to transfer BP-1311131906-172.17.0.2-1731847610939:blk_1073741853_1036 to 127.0.0.1:45021 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:13,649 WARN  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39533,DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b,DISK]]
2024-11-17T12:47:13,649 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:13,649 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0d751a622496%2C43029%2C1731847612705:(num 1731847629629) roll requested
2024-11-17T12:47:13,650 INFO  [regionserver/0d751a622496:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C43029%2C1731847612705.1731847633650
2024-11-17T12:47:13,656 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36351
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:13,656 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:45502 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741854_1037] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data8]'}, localName='127.0.0.1:39533', datanodeUuid='6bb64b52-41b8-49af-bf13-ba9bcad2229b', xmitsInProgress=0}:Exception transferring block BP-1311131906-172.17.0.2-1731847610939:blk_1073741854_1037 to mirror 127.0.0.1:36351
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:13,656 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39533,DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b,DISK], DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]) is bad.
2024-11-17T12:47:13,656 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741854_1037
2024-11-17T12:47:13,656 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:45502 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741854_1037] {}] datanode.BlockReceiver(316): Block 1073741854 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-17T12:47:13,656 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:45502 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741854_1037] {}] datanode.DataXceiver(331): 127.0.0.1:39533:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45502 dst: /127.0.0.1:39533
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:13,657 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]
2024-11-17T12:47:13,659 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:13,660 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK], DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]) is bad.
2024-11-17T12:47:13,660 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741855_1038
2024-11-17T12:47:13,661 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]
2024-11-17T12:47:13,663 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:13,663 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK], DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]) is bad.
2024-11-17T12:47:13,663 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741856_1039
2024-11-17T12:47:13,664 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]
2024-11-17T12:47:13,666 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37111
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:13,666 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:45510 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741857_1040] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data8]'}, localName='127.0.0.1:39533', datanodeUuid='6bb64b52-41b8-49af-bf13-ba9bcad2229b', xmitsInProgress=0}:Exception transferring block BP-1311131906-172.17.0.2-1731847610939:blk_1073741857_1040 to mirror 127.0.0.1:37111
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:13,666 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39533,DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b,DISK], DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]) is bad.
2024-11-17T12:47:13,666 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:45510 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741857_1040] {}] datanode.BlockReceiver(316): Block 1073741857 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-17T12:47:13,666 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741857_1040
2024-11-17T12:47:13,667 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:45510 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741857_1040] {}] datanode.DataXceiver(331): 127.0.0.1:39533:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45510 dst: /127.0.0.1:39533
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:13,667 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]
2024-11-17T12:47:13,668 WARN  [IPC Server handler 2 on default port 42079 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-17T12:47:13,668 WARN  [IPC Server handler 2 on default port 42079 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-17T12:47:13,668 WARN  [IPC Server handler 2 on default port 42079 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-17T12:47:13,671 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:13,671 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:13,671 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:13,671 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:13,671 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:13,672 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847629629 with entries=25, filesize=25.38 KB; new WAL /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847633650
2024-11-17T12:47:13,672 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40685:40685)]
2024-11-17T12:47:13,672 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847613367 is not closed yet, will try archiving it next time
2024-11-17T12:47:13,672 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847629629 is not closed yet, will try archiving it next time
2024-11-17T12:47:13,673 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847625614 to hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/oldWALs/0d751a622496%2C43029%2C1731847612705.1731847625614
2024-11-17T12:47:13,673 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741843_1026 (size=25992)
2024-11-17T12:47:13,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43029 {}] regionserver.HRegion(8855): Flush requested on dc2b8c9bdc7b25edc69ab039c0619f99
2024-11-17T12:47:13,764 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc2b8c9bdc7b25edc69ab039c0619f99 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-17T12:47:13,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/.tmp/info/d42ab62b652e4baf982503554f105fe8 is 1079, key is tmprow/info:/1731847633761/Put/seqid=0
2024-11-17T12:47:13,773 WARN  [Thread-941 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:13,774 WARN  [Thread-941 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK], DatanodeInfoWithStorage[127.0.0.1:39533,DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]) is bad.
2024-11-17T12:47:13,774 WARN  [Thread-941 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741859_1042
2024-11-17T12:47:13,774 WARN  [Thread-941 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]
2024-11-17T12:47:13,776 WARN  [Thread-941 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:13,776 WARN  [Thread-941 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK], DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]) is bad.
2024-11-17T12:47:13,776 WARN  [Thread-941 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741860_1043
2024-11-17T12:47:13,777 WARN  [Thread-941 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]
2024-11-17T12:47:13,779 WARN  [Thread-941 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:13,779 WARN  [Thread-941 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK], DatanodeInfoWithStorage[127.0.0.1:39533,DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]) is bad.
2024-11-17T12:47:13,779 WARN  [Thread-941 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741861_1044
2024-11-17T12:47:13,780 WARN  [Thread-941 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]
2024-11-17T12:47:13,781 WARN  [Thread-941 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:13,782 WARN  [Thread-941 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK], DatanodeInfoWithStorage[127.0.0.1:39533,DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]) is bad.
2024-11-17T12:47:13,782 WARN  [Thread-941 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741862_1045
2024-11-17T12:47:13,782 WARN  [Thread-941 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]
2024-11-17T12:47:13,783 WARN  [IPC Server handler 1 on default port 42079 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-17T12:47:13,784 WARN  [IPC Server handler 1 on default port 42079 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-17T12:47:13,784 WARN  [IPC Server handler 1 on default port 42079 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-17T12:47:13,787 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741863_1046 (size=6027)
2024-11-17T12:47:14,074 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847613367 is not closed yet, will try archiving it next time
2024-11-17T12:47:14,092 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:14,189 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/.tmp/info/d42ab62b652e4baf982503554f105fe8
2024-11-17T12:47:14,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/.tmp/info/d42ab62b652e4baf982503554f105fe8 as hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/d42ab62b652e4baf982503554f105fe8
2024-11-17T12:47:14,210 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/d42ab62b652e4baf982503554f105fe8, entries=1, sequenceid=34, filesize=5.9 K
2024-11-17T12:47:14,212 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for dc2b8c9bdc7b25edc69ab039c0619f99 in 448ms, sequenceid=34, compaction requested=true
2024-11-17T12:47:14,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc2b8c9bdc7b25edc69ab039c0619f99:

2024-11-17T12:47:14,212 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K
2024-11-17T12:47:14,212 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:47:14,212 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/fc2a95a1cad1442ea020ad7e66fc262f because midkey is the same as first or last row
2024-11-17T12:47:14,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc2b8c9bdc7b25edc69ab039c0619f99:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T12:47:14,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:47:14,213 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T12:47:14,214 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-17T12:47:14,214 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.HStore(1541): dc2b8c9bdc7b25edc69ab039c0619f99/info is initiating minor compaction (all files)
2024-11-17T12:47:14,214 INFO  [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc2b8c9bdc7b25edc69ab039c0619f99/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.
2024-11-17T12:47:14,215 INFO  [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/b492cab2e0a74b6ab93a360f50888d77, hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/fc2a95a1cad1442ea020ad7e66fc262f, hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/d42ab62b652e4baf982503554f105fe8] into tmpdir=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/.tmp, totalSize=28.2 K
2024-11-17T12:47:14,215 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] compactions.Compactor(225): Compacting b492cab2e0a74b6ab93a360f50888d77, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731847627653
2024-11-17T12:47:14,216 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc2a95a1cad1442ea020ad7e66fc262f, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1731847631716
2024-11-17T12:47:14,216 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] compactions.Compactor(225): Compacting d42ab62b652e4baf982503554f105fe8, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731847633761
2024-11-17T12:47:14,231 INFO  [RS:0;0d751a622496:43029-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc2b8c9bdc7b25edc69ab039c0619f99#info#compaction#21 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T12:47:14,232 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/.tmp/info/41511714a5c241908c9d9462f096e81a is 1080, key is row0002/info:/1731847627653/Put/seqid=0
2024-11-17T12:47:14,234 WARN  [Thread-947 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1047
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:14,234 WARN  [Thread-947 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741864_1047 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK], DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]) is bad.
2024-11-17T12:47:14,234 WARN  [Thread-947 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741864_1047
2024-11-17T12:47:14,235 WARN  [Thread-947 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]
2024-11-17T12:47:14,237 WARN  [Thread-947 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741865_1048
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37111
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:14,237 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:45548 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741865_1048] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data8]'}, localName='127.0.0.1:39533', datanodeUuid='6bb64b52-41b8-49af-bf13-ba9bcad2229b', xmitsInProgress=0}:Exception transferring block BP-1311131906-172.17.0.2-1731847610939:blk_1073741865_1048 to mirror 127.0.0.1:37111
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:14,237 WARN  [Thread-947 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741865_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39533,DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b,DISK], DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]) is bad.
2024-11-17T12:47:14,237 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:45548 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741865_1048] {}] datanode.BlockReceiver(316): Block 1073741865 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-17T12:47:14,237 WARN  [Thread-947 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741865_1048
2024-11-17T12:47:14,237 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:45548 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741865_1048] {}] datanode.DataXceiver(331): 127.0.0.1:39533:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45548 dst: /127.0.0.1:39533
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:14,238 WARN  [Thread-947 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]
2024-11-17T12:47:14,239 WARN  [Thread-947 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:14,239 WARN  [Thread-947 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741866_1049 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK], DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]) is bad.
2024-11-17T12:47:14,239 WARN  [Thread-947 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741866_1049
2024-11-17T12:47:14,240 WARN  [Thread-947 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]
2024-11-17T12:47:14,241 WARN  [Thread-947 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741867_1050
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:14,241 WARN  [Thread-947 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741867_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK], DatanodeInfoWithStorage[127.0.0.1:39533,DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]) is bad.
2024-11-17T12:47:14,241 WARN  [Thread-947 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741867_1050
2024-11-17T12:47:14,242 WARN  [Thread-947 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]
2024-11-17T12:47:14,242 WARN  [IPC Server handler 0 on default port 42079 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-17T12:47:14,242 WARN  [IPC Server handler 0 on default port 42079 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-17T12:47:14,242 WARN  [IPC Server handler 0 on default port 42079 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-17T12:47:14,246 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741868_1051 (size=17994)
2024-11-17T12:47:14,657 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/.tmp/info/41511714a5c241908c9d9462f096e81a as hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/41511714a5c241908c9d9462f096e81a
2024-11-17T12:47:14,666 INFO  [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dc2b8c9bdc7b25edc69ab039c0619f99/info of dc2b8c9bdc7b25edc69ab039c0619f99 into 41511714a5c241908c9d9462f096e81a(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T12:47:14,666 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc2b8c9bdc7b25edc69ab039c0619f99:

2024-11-17T12:47:14,667 INFO  [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99., storeName=dc2b8c9bdc7b25edc69ab039c0619f99/info, priority=13, startTime=1731847634212; duration=0sec
2024-11-17T12:47:14,667 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-17T12:47:14,667 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:47:14,667 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/41511714a5c241908c9d9462f096e81a because midkey is the same as first or last row
2024-11-17T12:47:14,667 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-17T12:47:14,667 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:47:14,667 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/41511714a5c241908c9d9462f096e81a because midkey is the same as first or last row
2024-11-17T12:47:14,667 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-17T12:47:14,667 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:47:14,667 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/41511714a5c241908c9d9462f096e81a because midkey is the same as first or last row
2024-11-17T12:47:14,667 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:47:14,668 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc2b8c9bdc7b25edc69ab039c0619f99:info
2024-11-17T12:47:14,819 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43029 {}] regionserver.HRegion(8855): Flush requested on dc2b8c9bdc7b25edc69ab039c0619f99
2024-11-17T12:47:15,200 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc2b8c9bdc7b25edc69ab039c0619f99 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-17T12:47:15,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/.tmp/info/59540e2e4d74409da39ec8f13efea42a is 1079, key is tmprow/info:/1731847635199/Put/seqid=0
2024-11-17T12:47:15,210 WARN  [Thread-952 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1052
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:15,210 WARN  [Thread-952 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741869_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK], DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]) is bad.
2024-11-17T12:47:15,210 WARN  [Thread-952 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741869_1052
2024-11-17T12:47:15,211 WARN  [Thread-952 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]
2024-11-17T12:47:15,213 WARN  [Thread-952 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741870_1053
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37111
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:15,213 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:45570 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741870_1053] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data8]'}, localName='127.0.0.1:39533', datanodeUuid='6bb64b52-41b8-49af-bf13-ba9bcad2229b', xmitsInProgress=0}:Exception transferring block BP-1311131906-172.17.0.2-1731847610939:blk_1073741870_1053 to mirror 127.0.0.1:37111
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:15,213 WARN  [Thread-952 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741870_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39533,DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b,DISK], DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]) is bad.
2024-11-17T12:47:15,213 WARN  [Thread-952 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741870_1053
2024-11-17T12:47:15,213 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:45570 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741870_1053] {}] datanode.BlockReceiver(316): Block 1073741870 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-17T12:47:15,214 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:45570 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741870_1053] {}] datanode.DataXceiver(331): 127.0.0.1:39533:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45570 dst: /127.0.0.1:39533
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:15,214 WARN  [Thread-952 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]
2024-11-17T12:47:15,215 WARN  [Thread-952 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741871_1054
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:15,215 WARN  [Thread-952 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741871_1054 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK], DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]) is bad.
2024-11-17T12:47:15,215 WARN  [Thread-952 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741871_1054
2024-11-17T12:47:15,216 WARN  [Thread-952 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]
2024-11-17T12:47:15,217 WARN  [Thread-952 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1055
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:15,217 WARN  [Thread-952 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741872_1055 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK], DatanodeInfoWithStorage[127.0.0.1:39533,DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]) is bad.
2024-11-17T12:47:15,217 WARN  [Thread-952 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741872_1055
2024-11-17T12:47:15,218 WARN  [Thread-952 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]
2024-11-17T12:47:15,218 WARN  [IPC Server handler 4 on default port 42079 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-17T12:47:15,218 WARN  [IPC Server handler 4 on default port 42079 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-17T12:47:15,218 WARN  [IPC Server handler 4 on default port 42079 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-17T12:47:15,221 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741873_1056 (size=6027)
2024-11-17T12:47:15,623 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/.tmp/info/59540e2e4d74409da39ec8f13efea42a
2024-11-17T12:47:15,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/.tmp/info/59540e2e4d74409da39ec8f13efea42a as hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/59540e2e4d74409da39ec8f13efea42a
2024-11-17T12:47:15,639 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/59540e2e4d74409da39ec8f13efea42a, entries=1, sequenceid=45, filesize=5.9 K
2024-11-17T12:47:15,640 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for dc2b8c9bdc7b25edc69ab039c0619f99 in 440ms, sequenceid=45, compaction requested=false
2024-11-17T12:47:15,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc2b8c9bdc7b25edc69ab039c0619f99:

2024-11-17T12:47:15,641 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K
2024-11-17T12:47:15,641 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:47:15,641 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/41511714a5c241908c9d9462f096e81a because midkey is the same as first or last row
2024-11-17T12:47:15,673 WARN  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39533,DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b,DISK]]
2024-11-17T12:47:15,673 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:15,673 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0d751a622496%2C43029%2C1731847612705:(num 1731847633650) roll requested
2024-11-17T12:47:15,674 INFO  [regionserver/0d751a622496:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C43029%2C1731847612705.1731847635673
2024-11-17T12:47:15,676 WARN  [Thread-957 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741874_1057
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:15,677 WARN  [Thread-957 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741874_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK], DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]) is bad.
2024-11-17T12:47:15,677 WARN  [Thread-957 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741874_1057
2024-11-17T12:47:15,677 WARN  [Thread-957 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]
2024-11-17T12:47:15,678 WARN  [Thread-957 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741875_1058
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:15,679 WARN  [Thread-957 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741875_1058 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK], DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]) is bad.
2024-11-17T12:47:15,679 WARN  [Thread-957 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741875_1058
2024-11-17T12:47:15,679 WARN  [Thread-957 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]
2024-11-17T12:47:15,681 WARN  [Thread-957 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741876_1059
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36351
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:15,681 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:45576 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741876_1059] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data8]'}, localName='127.0.0.1:39533', datanodeUuid='6bb64b52-41b8-49af-bf13-ba9bcad2229b', xmitsInProgress=0}:Exception transferring block BP-1311131906-172.17.0.2-1731847610939:blk_1073741876_1059 to mirror 127.0.0.1:36351
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:15,681 WARN  [Thread-957 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741876_1059 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39533,DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b,DISK], DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]) is bad.
2024-11-17T12:47:15,681 WARN  [Thread-957 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741876_1059
2024-11-17T12:47:15,681 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:45576 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741876_1059] {}] datanode.BlockReceiver(316): Block 1073741876 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-17T12:47:15,681 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:45576 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741876_1059] {}] datanode.DataXceiver(331): 127.0.0.1:39533:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45576 dst: /127.0.0.1:39533
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:15,682 WARN  [Thread-957 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]
2024-11-17T12:47:15,684 WARN  [Thread-957 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741877_1060
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:45021
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:15,684 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:45578 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741877_1060] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data8]'}, localName='127.0.0.1:39533', datanodeUuid='6bb64b52-41b8-49af-bf13-ba9bcad2229b', xmitsInProgress=0}:Exception transferring block BP-1311131906-172.17.0.2-1731847610939:blk_1073741877_1060 to mirror 127.0.0.1:45021
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:15,684 WARN  [Thread-957 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741877_1060 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39533,DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b,DISK], DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]) is bad.
2024-11-17T12:47:15,684 WARN  [Thread-957 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741877_1060
2024-11-17T12:47:15,684 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:45578 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741877_1060] {}] datanode.BlockReceiver(316): Block 1073741877 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-17T12:47:15,684 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:45578 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741877_1060] {}] datanode.DataXceiver(331): 127.0.0.1:39533:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45578 dst: /127.0.0.1:39533
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:15,685 WARN  [Thread-957 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]
2024-11-17T12:47:15,685 WARN  [IPC Server handler 4 on default port 42079 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-17T12:47:15,685 WARN  [IPC Server handler 4 on default port 42079 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-17T12:47:15,685 WARN  [IPC Server handler 4 on default port 42079 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-17T12:47:15,688 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:15,688 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:15,688 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:15,688 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:15,688 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:15,688 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847633650 with entries=15, filesize=13.26 KB; new WAL /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847635673
2024-11-17T12:47:15,689 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40685:40685)]
2024-11-17T12:47:15,689 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847613367 is not closed yet, will try archiving it next time
2024-11-17T12:47:15,689 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847633650 is not closed yet, will try archiving it next time
2024-11-17T12:47:15,689 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847629629 to hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/oldWALs/0d751a622496%2C43029%2C1731847612705.1731847629629
2024-11-17T12:47:15,690 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741858_1041 (size=13591)
2024-11-17T12:47:16,092 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847613367 is not closed yet, will try archiving it next time
2024-11-17T12:47:16,092 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:16,432 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4c81f1c7[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39533, datanodeUuid=6bb64b52-41b8-49af-bf13-ba9bcad2229b, infoPort=40685, infoSecurePort=0, ipcPort=41737, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939):Failed to transfer BP-1311131906-172.17.0.2-1731847610939:blk_1073741863_1046 to 127.0.0.1:33741 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:16,432 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@2bea8b4b[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39533, datanodeUuid=6bb64b52-41b8-49af-bf13-ba9bcad2229b, infoPort=40685, infoSecurePort=0, ipcPort=41737, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939):Failed to transfer BP-1311131906-172.17.0.2-1731847610939:blk_1073741843_1026 to 127.0.0.1:33741 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:16,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43029 {}] regionserver.HRegion(8855): Flush requested on dc2b8c9bdc7b25edc69ab039c0619f99
2024-11-17T12:47:16,633 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc2b8c9bdc7b25edc69ab039c0619f99 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-17T12:47:16,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/.tmp/info/aeca021bcdf34a7ba78782f07258c715 is 1079, key is tmprow/info:/1731847636631/Put/seqid=0
2024-11-17T12:47:16,645 WARN  [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741879_1062
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37111
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:16,645 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:45584 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741879_1062] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data8]'}, localName='127.0.0.1:39533', datanodeUuid='6bb64b52-41b8-49af-bf13-ba9bcad2229b', xmitsInProgress=0}:Exception transferring block BP-1311131906-172.17.0.2-1731847610939:blk_1073741879_1062 to mirror 127.0.0.1:37111
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:16,646 WARN  [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741879_1062 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39533,DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b,DISK], DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]) is bad.
2024-11-17T12:47:16,646 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:45584 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741879_1062] {}] datanode.BlockReceiver(316): Block 1073741879 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-17T12:47:16,646 WARN  [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741879_1062
2024-11-17T12:47:16,646 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:45584 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741879_1062] {}] datanode.DataXceiver(331): 127.0.0.1:39533:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45584 dst: /127.0.0.1:39533
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:16,647 WARN  [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]
2024-11-17T12:47:16,648 WARN  [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741880_1063
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:16,649 WARN  [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741880_1063 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK], DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]) is bad.
2024-11-17T12:47:16,649 WARN  [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741880_1063
2024-11-17T12:47:16,649 WARN  [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]
2024-11-17T12:47:16,651 WARN  [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741881_1064
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:16,651 WARN  [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741881_1064 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK], DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]) is bad.
2024-11-17T12:47:16,651 WARN  [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741881_1064
2024-11-17T12:47:16,651 WARN  [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]
2024-11-17T12:47:16,653 WARN  [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741882_1065
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:16,653 WARN  [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741882_1065 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK], DatanodeInfoWithStorage[127.0.0.1:39533,DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]) is bad.
2024-11-17T12:47:16,653 WARN  [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741882_1065
2024-11-17T12:47:16,654 WARN  [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]
2024-11-17T12:47:16,655 WARN  [IPC Server handler 4 on default port 42079 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-17T12:47:16,655 WARN  [IPC Server handler 4 on default port 42079 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-17T12:47:16,655 WARN  [IPC Server handler 4 on default port 42079 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-17T12:47:16,658 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741883_1066 (size=6027)
2024-11-17T12:47:16,820 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:17,059 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/.tmp/info/aeca021bcdf34a7ba78782f07258c715
2024-11-17T12:47:17,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/.tmp/info/aeca021bcdf34a7ba78782f07258c715 as hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/aeca021bcdf34a7ba78782f07258c715
2024-11-17T12:47:17,074 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/aeca021bcdf34a7ba78782f07258c715, entries=1, sequenceid=55, filesize=5.9 K
2024-11-17T12:47:17,075 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for dc2b8c9bdc7b25edc69ab039c0619f99 in 442ms, sequenceid=55, compaction requested=true
2024-11-17T12:47:17,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc2b8c9bdc7b25edc69ab039c0619f99:

2024-11-17T12:47:17,075 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=29.3 K, sizeToCheck=16.0 K
2024-11-17T12:47:17,075 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:47:17,075 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/41511714a5c241908c9d9462f096e81a because midkey is the same as first or last row
2024-11-17T12:47:17,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc2b8c9bdc7b25edc69ab039c0619f99:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T12:47:17,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:47:17,076 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T12:47:17,077 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 30048 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-17T12:47:17,077 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.HStore(1541): dc2b8c9bdc7b25edc69ab039c0619f99/info is initiating minor compaction (all files)
2024-11-17T12:47:17,077 INFO  [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc2b8c9bdc7b25edc69ab039c0619f99/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.
2024-11-17T12:47:17,077 INFO  [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/41511714a5c241908c9d9462f096e81a, hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/59540e2e4d74409da39ec8f13efea42a, hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/aeca021bcdf34a7ba78782f07258c715] into tmpdir=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/.tmp, totalSize=29.3 K
2024-11-17T12:47:17,078 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41511714a5c241908c9d9462f096e81a, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731847627653
2024-11-17T12:47:17,078 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59540e2e4d74409da39ec8f13efea42a, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731847635199
2024-11-17T12:47:17,079 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] compactions.Compactor(225): Compacting aeca021bcdf34a7ba78782f07258c715, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731847636631
2024-11-17T12:47:17,094 INFO  [RS:0;0d751a622496:43029-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc2b8c9bdc7b25edc69ab039c0619f99#info#compaction#24 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T12:47:17,094 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/.tmp/info/dfd6d1885169476a82e83a0fc12feef1 is 1080, key is row0002/info:/1731847627653/Put/seqid=0
2024-11-17T12:47:17,096 WARN  [Thread-969 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741884_1067
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:17,096 WARN  [Thread-969 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741884_1067 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK], DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]) is bad.
2024-11-17T12:47:17,096 WARN  [Thread-969 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741884_1067
2024-11-17T12:47:17,097 WARN  [Thread-969 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]
2024-11-17T12:47:17,098 WARN  [Thread-969 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741885_1068
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:17,098 WARN  [Thread-969 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741885_1068 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK], DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]) is bad.
2024-11-17T12:47:17,098 WARN  [Thread-969 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741885_1068
2024-11-17T12:47:17,099 WARN  [Thread-969 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]
2024-11-17T12:47:17,100 WARN  [Thread-969 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741886_1069
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:17,101 WARN  [Thread-969 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741886_1069 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK], DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]) is bad.
2024-11-17T12:47:17,101 WARN  [Thread-969 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741886_1069
2024-11-17T12:47:17,101 WARN  [Thread-969 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]
2024-11-17T12:47:17,103 WARN  [Thread-969 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741887_1070
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33741
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:17,103 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:45602 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741887_1070] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data8]'}, localName='127.0.0.1:39533', datanodeUuid='6bb64b52-41b8-49af-bf13-ba9bcad2229b', xmitsInProgress=0}:Exception transferring block BP-1311131906-172.17.0.2-1731847610939:blk_1073741887_1070 to mirror 127.0.0.1:33741
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:17,103 WARN  [Thread-969 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741887_1070 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39533,DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b,DISK], DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]) is bad.
2024-11-17T12:47:17,103 WARN  [Thread-969 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741887_1070
2024-11-17T12:47:17,103 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:45602 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741887_1070] {}] datanode.BlockReceiver(316): Block 1073741887 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-17T12:47:17,104 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:45602 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741887_1070] {}] datanode.DataXceiver(331): 127.0.0.1:39533:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:45602 dst: /127.0.0.1:39533
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:17,104 WARN  [Thread-969 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33741,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]
2024-11-17T12:47:17,105 WARN  [IPC Server handler 4 on default port 42079 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-17T12:47:17,105 WARN  [IPC Server handler 4 on default port 42079 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-17T12:47:17,105 WARN  [IPC Server handler 4 on default port 42079 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-17T12:47:17,108 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741888_1071 (size=18097)
2024-11-17T12:47:17,432 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@2bea8b4b[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39533, datanodeUuid=6bb64b52-41b8-49af-bf13-ba9bcad2229b, infoPort=40685, infoSecurePort=0, ipcPort=41737, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939):Failed to transfer BP-1311131906-172.17.0.2-1731847610939:blk_1073741868_1051 to 127.0.0.1:36351 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:17,432 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4c81f1c7[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39533, datanodeUuid=6bb64b52-41b8-49af-bf13-ba9bcad2229b, infoPort=40685, infoSecurePort=0, ipcPort=41737, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939):Failed to transfer BP-1311131906-172.17.0.2-1731847610939:blk_1073741873_1056 to 127.0.0.1:36351 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:17,524 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/.tmp/info/dfd6d1885169476a82e83a0fc12feef1 as hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/dfd6d1885169476a82e83a0fc12feef1
2024-11-17T12:47:17,532 INFO  [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dc2b8c9bdc7b25edc69ab039c0619f99/info of dc2b8c9bdc7b25edc69ab039c0619f99 into dfd6d1885169476a82e83a0fc12feef1(size=17.7 K), total size for store is 17.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T12:47:17,532 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc2b8c9bdc7b25edc69ab039c0619f99:

2024-11-17T12:47:17,532 INFO  [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99., storeName=dc2b8c9bdc7b25edc69ab039c0619f99/info, priority=13, startTime=1731847637075; duration=0sec
2024-11-17T12:47:17,532 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-17T12:47:17,532 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:47:17,532 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/dfd6d1885169476a82e83a0fc12feef1 because midkey is the same as first or last row
2024-11-17T12:47:17,532 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-17T12:47:17,532 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:47:17,532 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/dfd6d1885169476a82e83a0fc12feef1 because midkey is the same as first or last row
2024-11-17T12:47:17,533 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-17T12:47:17,533 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:47:17,533 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/dfd6d1885169476a82e83a0fc12feef1 because midkey is the same as first or last row
2024-11-17T12:47:17,533 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:47:17,533 DEBUG [RS:0;0d751a622496:43029-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc2b8c9bdc7b25edc69ab039c0619f99:info
2024-11-17T12:47:17,689 WARN  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas.
2024-11-17T12:47:17,689 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:17,863 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:47:17,868 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:47:17,871 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:47:17,871 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:47:17,871 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T12:47:17,872 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@796aacdc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:47:17,873 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3edaa0c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:47:17,967 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@30d4c51a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/java.io.tmpdir/jetty-localhost-33891-hadoop-hdfs-3_4_1-tests_jar-_-any-13425447551928990165/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:47:17,968 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34827478{HTTP/1.1, (http/1.1)}{localhost:33891}
2024-11-17T12:47:17,968 INFO  [Time-limited test {}] server.Server(415): Started @133209ms
2024-11-17T12:47:17,969 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T12:47:18,093 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:18,361 WARN  [Thread-988 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T12:47:18,368 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd6404a9acc06ba90 with lease ID 0xfc001fc3e00951e: from storage DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b node DatanodeRegistration(127.0.0.1:39105, datanodeUuid=368a0fe1-98fb-4e11-a7ad-3861a4d4f891, infoPort=34593, infoSecurePort=0, ipcPort=40369, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:47:18,368 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd6404a9acc06ba90 with lease ID 0xfc001fc3e00951e: from storage DS-82a38059-b18c-440f-9aea-207b1fa279b2 node DatanodeRegistration(127.0.0.1:39105, datanodeUuid=368a0fe1-98fb-4e11-a7ad-3861a4d4f891, infoPort=34593, infoSecurePort=0, ipcPort=40369, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-17T12:47:18,820 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:19,433 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4c81f1c7[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39533, datanodeUuid=6bb64b52-41b8-49af-bf13-ba9bcad2229b, infoPort=40685, infoSecurePort=0, ipcPort=41737, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939):Failed to transfer BP-1311131906-172.17.0.2-1731847610939:blk_1073741858_1041 to 127.0.0.1:36351 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:19,438 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39105 is added to blk_1073741883_1066 (size=6027)
2024-11-17T12:47:19,690 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:20,093 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:20,433 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@2bea8b4b[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39533, datanodeUuid=6bb64b52-41b8-49af-bf13-ba9bcad2229b, infoPort=40685, infoSecurePort=0, ipcPort=41737, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939):Failed to transfer BP-1311131906-172.17.0.2-1731847610939:blk_1073741888_1071 to 127.0.0.1:36351 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:20,821 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:21,690 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:22,094 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:22,532 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-17T12:47:22,821 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:23,062 ERROR [FSHLog-0-hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData-prefix:0d751a622496,37969,1731847612555 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:23,062 WARN  [FSHLog-0-hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData-prefix:0d751a622496,37969,1731847612555 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:23,063 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog 0d751a622496%2C37969%2C1731847612555:(num 1731847612841) roll requested
2024-11-17T12:47:23,064 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C37969%2C1731847612555.1731847643063
2024-11-17T12:47:23,070 WARN  [Thread-1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741889_1072
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36351
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:23,070 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_475526744_22 at /127.0.0.1:40632 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741889_1072] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data8]'}, localName='127.0.0.1:39533', datanodeUuid='6bb64b52-41b8-49af-bf13-ba9bcad2229b', xmitsInProgress=0}:Exception transferring block BP-1311131906-172.17.0.2-1731847610939:blk_1073741889_1072 to mirror 127.0.0.1:36351
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:23,070 WARN  [Thread-1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741889_1072 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39533,DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b,DISK], DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]) is bad.
2024-11-17T12:47:23,070 WARN  [Thread-1009 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741889_1072
2024-11-17T12:47:23,070 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_475526744_22 at /127.0.0.1:40632 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741889_1072] {}] datanode.BlockReceiver(316): Block 1073741889 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-17T12:47:23,070 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_475526744_22 at /127.0.0.1:40632 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741889_1072] {}] datanode.DataXceiver(331): 127.0.0.1:39533:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40632 dst: /127.0.0.1:39533
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:23,071 WARN  [Thread-1009 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]
2024-11-17T12:47:23,074 WARN  [Thread-1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741890_1073
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:37111
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:23,074 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_475526744_22 at /127.0.0.1:44762 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741890_1073] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data4]'}, localName='127.0.0.1:39105', datanodeUuid='368a0fe1-98fb-4e11-a7ad-3861a4d4f891', xmitsInProgress=0}:Exception transferring block BP-1311131906-172.17.0.2-1731847610939:blk_1073741890_1073 to mirror 127.0.0.1:37111
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:23,074 WARN  [Thread-1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741890_1073 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39105,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK], DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]) is bad.
2024-11-17T12:47:23,074 WARN  [Thread-1009 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741890_1073
2024-11-17T12:47:23,074 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_475526744_22 at /127.0.0.1:44762 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741890_1073] {}] datanode.BlockReceiver(316): Block 1073741890 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-17T12:47:23,075 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_475526744_22 at /127.0.0.1:44762 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741890_1073] {}] datanode.DataXceiver(331): 127.0.0.1:39105:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:44762 dst: /127.0.0.1:39105
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:23,075 WARN  [Thread-1009 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:37111,DS-161e2e36-93ec-4012-b9bc-7865bcb4b452,DISK]
2024-11-17T12:47:23,080 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:23,080 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:23,080 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:23,080 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:23,080 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:23,080 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/WALs/0d751a622496,37969,1731847612555/0d751a622496%2C37969%2C1731847612555.1731847612841 with entries=54, filesize=26.67 KB; new WAL /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/WALs/0d751a622496,37969,1731847612555/0d751a622496%2C37969%2C1731847612555.1731847643063
2024-11-17T12:47:23,081 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:23,081 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:23,081 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/WALs/0d751a622496,37969,1731847612555/0d751a622496%2C37969%2C1731847612555.1731847612841
2024-11-17T12:47:23,081 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40685:40685),(127.0.0.1/127.0.0.1:34593:34593)]
2024-11-17T12:47:23,081 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/WALs/0d751a622496,37969,1731847612555/0d751a622496%2C37969%2C1731847612555.1731847612841 is not closed yet, will try archiving it next time
2024-11-17T12:47:23,082 WARN  [IPC Server handler 2 on default port 42079 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/WALs/0d751a622496,37969,1731847612555/0d751a622496%2C37969%2C1731847612555.1731847612841 has not been closed. Lease recovery is in progress. RecoveryId = 1075 for block blk_1073741830_1006
2024-11-17T12:47:23,082 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/WALs/0d751a622496,37969,1731847612555/0d751a622496%2C37969%2C1731847612555.1731847612841 after 1ms
2024-11-17T12:47:23,691 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:24,094 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:25,691 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:26,095 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:27,084 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/WALs/0d751a622496,37969,1731847612555/0d751a622496%2C37969%2C1731847612555.1731847612841 after 4003ms
2024-11-17T12:47:27,692 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:28,095 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:28,366 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@28152aad[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39105, datanodeUuid=368a0fe1-98fb-4e11-a7ad-3861a4d4f891, infoPort=34593, infoSecurePort=0, ipcPort=40369, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939):Failed to transfer BP-1311131906-172.17.0.2-1731847610939:blk_1073741835_1011 to 127.0.0.1:37111 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:28,368 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@5346f772[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39105, datanodeUuid=368a0fe1-98fb-4e11-a7ad-3861a4d4f891, infoPort=34593, infoSecurePort=0, ipcPort=40369, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939):Failed to transfer BP-1311131906-172.17.0.2-1731847610939:blk_1073741831_1007 to 127.0.0.1:37111 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:28,385 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@3369083d {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1311131906-172.17.0.2-1731847610939:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:45021,null,null])
java.net.ConnectException: Call From 0d751a622496/172.17.0.2 to localhost:33121 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-17T12:47:28,387 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39105 is added to blk_1073741833_1020 (size=455)
2024-11-17T12:47:28,659 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847613367 to hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/oldWALs/0d751a622496%2C43029%2C1731847612705.1731847613367
2024-11-17T12:47:28,660 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847633650 to hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/oldWALs/0d751a622496%2C43029%2C1731847612705.1731847633650
2024-11-17T12:47:29,368 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741827_1003 (size=196)
2024-11-17T12:47:29,368 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741829_1005 (size=34)
2024-11-17T12:47:29,692 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:30,096 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:31,367 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@5346f772[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39105, datanodeUuid=368a0fe1-98fb-4e11-a7ad-3861a4d4f891, infoPort=34593, infoSecurePort=0, ipcPort=40369, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939):Failed to transfer BP-1311131906-172.17.0.2-1731847610939:blk_1073741826_1002 to 127.0.0.1:36351 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:31,369 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741833_1020 (size=455)
2024-11-17T12:47:31,648 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C43029%2C1731847612705.1731847651648
2024-11-17T12:47:31,651 WARN  [Thread-1026 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741892_1076
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:31,651 WARN  [Thread-1026 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741892_1076 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK], DatanodeInfoWithStorage[127.0.0.1:39105,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]) is bad.
2024-11-17T12:47:31,651 WARN  [Thread-1026 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741892_1076
2024-11-17T12:47:31,652 WARN  [Thread-1026 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]
2024-11-17T12:47:31,661 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:31,661 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:31,661 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:31,661 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:31,661 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:31,661 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847635673 with entries=13, filesize=12.60 KB; new WAL /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847651648
2024-11-17T12:47:31,663 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741878_1061 (size=12911)
2024-11-17T12:47:31,664 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40685:40685),(127.0.0.1/127.0.0.1:34593:34593)]
2024-11-17T12:47:31,664 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847635673 is not closed yet, will try archiving it next time
2024-11-17T12:47:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43029 {}] regionserver.HRegion(8855): Flush requested on dc2b8c9bdc7b25edc69ab039c0619f99
2024-11-17T12:47:31,669 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc2b8c9bdc7b25edc69ab039c0619f99 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-17T12:47:31,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/.tmp/info/c2e55159c1d34f29acb0b1d3a44b70d9 is 1080, key is row0013/info:/1731847651665/Put/seqid=0
2024-11-17T12:47:31,676 WARN  [Thread-1032 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741894_1078
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:31,676 WARN  [Thread-1032 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741894_1078 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK], DatanodeInfoWithStorage[127.0.0.1:39533,DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]) is bad.
2024-11-17T12:47:31,676 WARN  [Thread-1032 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741894_1078
2024-11-17T12:47:31,677 WARN  [Thread-1032 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]
2024-11-17T12:47:31,683 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741895_1079 (size=8190)
2024-11-17T12:47:31,683 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39105 is added to blk_1073741895_1079 (size=8190)
2024-11-17T12:47:31,684 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/.tmp/info/c2e55159c1d34f29acb0b1d3a44b70d9
2024-11-17T12:47:31,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/.tmp/info/c2e55159c1d34f29acb0b1d3a44b70d9 as hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/c2e55159c1d34f29acb0b1d3a44b70d9
2024-11-17T12:47:31,693 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled.
2024-11-17T12:47:31,693 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:31,696 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-17T12:47:31,696 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T12:47:31,696 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T12:47:31,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:47:31,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:47:31,697 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-17T12:47:31,697 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-17T12:47:31,697 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=401360406, stopped=false
2024-11-17T12:47:31,697 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0d751a622496,37969,1731847612555
2024-11-17T12:47:31,700 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/c2e55159c1d34f29acb0b1d3a44b70d9, entries=3, sequenceid=66, filesize=8.0 K
2024-11-17T12:47:31,701 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7527, heapSize ~8.11 KB/8304, currentSize=8.41 KB/8608 for dc2b8c9bdc7b25edc69ab039c0619f99 in 32ms, sequenceid=66, compaction requested=false
2024-11-17T12:47:31,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc2b8c9bdc7b25edc69ab039c0619f99:

2024-11-17T12:47:31,702 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=25.7 K, sizeToCheck=16.0 K
2024-11-17T12:47:31,702 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:47:31,702 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/dfd6d1885169476a82e83a0fc12feef1 because midkey is the same as first or last row
2024-11-17T12:47:31,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37689-0x1014a1efb3c0002, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T12:47:31,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T12:47:31,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:47:31,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37689-0x1014a1efb3c0002, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:47:31,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43029-0x1014a1efb3c0001, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T12:47:31,756 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T12:47:31,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43029-0x1014a1efb3c0001, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:47:31,756 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T12:47:31,756 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T12:47:31,757 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:47:31,757 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43029-0x1014a1efb3c0001, quorum=127.0.0.1:62721, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:47:31,757 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0d751a622496,43029,1731847612705' *****
2024-11-17T12:47:31,757 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-17T12:47:31,757 INFO  [RS:0;0d751a622496:43029 {}] regionserver.HRegionServer(878): Closing user regions
2024-11-17T12:47:31,757 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0d751a622496,37689,1731847613976' *****
2024-11-17T12:47:31,757 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-17T12:47:31,757 INFO  [RS:0;0d751a622496:43029 {}] regionserver.HRegionServer(3091): Received CLOSE for dc2b8c9bdc7b25edc69ab039c0619f99
2024-11-17T12:47:31,757 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37689-0x1014a1efb3c0002, quorum=127.0.0.1:62721, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:47:31,757 INFO  [RS:1;0d751a622496:37689 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-17T12:47:31,758 INFO  [RS:1;0d751a622496:37689 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-17T12:47:31,758 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-17T12:47:31,758 INFO  [RS:1;0d751a622496:37689 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-17T12:47:31,758 INFO  [RS:1;0d751a622496:37689 {}] regionserver.HRegionServer(959): stopping server 0d751a622496,37689,1731847613976
2024-11-17T12:47:31,758 INFO  [RS:1;0d751a622496:37689 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T12:47:31,758 INFO  [RS:1;0d751a622496:37689 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;0d751a622496:37689.
2024-11-17T12:47:31,758 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:47:31,758 DEBUG [RS:1;0d751a622496:37689 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T12:47:31,758 DEBUG [RS:1;0d751a622496:37689 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:47:31,758 INFO  [RS:1;0d751a622496:37689 {}] regionserver.HRegionServer(976): stopping server 0d751a622496,37689,1731847613976; all regions closed.
2024-11-17T12:47:31,758 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dc2b8c9bdc7b25edc69ab039c0619f99, disabling compactions & flushes
2024-11-17T12:47:31,758 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.
2024-11-17T12:47:31,758 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.
2024-11-17T12:47:31,759 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99. after waiting 0 ms
2024-11-17T12:47:31,759 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.
2024-11-17T12:47:31,759 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing dc2b8c9bdc7b25edc69ab039c0619f99 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB
2024-11-17T12:47:31,759 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:31,759 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:31,759 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:31,759 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:31,760 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:31,760 INFO  [RS:0;0d751a622496:43029 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-17T12:47:31,760 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:31,760 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-17T12:47:31,760 INFO  [RS:0;0d751a622496:43029 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-17T12:47:31,761 INFO  [RS:0;0d751a622496:43029 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-17T12:47:31,760 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:31,761 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
2024-11-17T12:47:31,761 INFO  [RS:0;0d751a622496:43029 {}] regionserver.HRegionServer(959): stopping server 0d751a622496,43029,1731847612705
2024-11-17T12:47:31,761 INFO  [RS:0;0d751a622496:43029 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T12:47:31,761 INFO  [RS:0;0d751a622496:43029 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0d751a622496:43029.
2024-11-17T12:47:31,761 DEBUG [RS:0;0d751a622496:43029 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T12:47:31,761 DEBUG [RS:0;0d751a622496:43029 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:47:31,761 INFO  [RS:0;0d751a622496:43029 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-17T12:47:31,761 INFO  [RS:0;0d751a622496:43029 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-17T12:47:31,761 WARN  [IPC Server handler 3 on default port 42079 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224 has not been closed. Lease recovery is in progress. RecoveryId = 1080 for block blk_1073741837_1013
2024-11-17T12:47:31,761 INFO  [RS:0;0d751a622496:43029 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-17T12:47:31,761 INFO  [RS:0;0d751a622496:43029 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-17T12:47:31,761 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224 after 0ms
2024-11-17T12:47:31,762 INFO  [RS:0;0d751a622496:43029 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-17T12:47:31,762 DEBUG [RS:0;0d751a622496:43029 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, dc2b8c9bdc7b25edc69ab039c0619f99=TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.}
2024-11-17T12:47:31,762 DEBUG [RS:0;0d751a622496:43029 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, dc2b8c9bdc7b25edc69ab039c0619f99
2024-11-17T12:47:31,762 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T12:47:31,762 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T12:47:31,762 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T12:47:31,762 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T12:47:31,762 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T12:47:31,762 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB
2024-11-17T12:47:31,763 ERROR [FSHLog-0-hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8-prefix:0d751a622496,43029,1731847612705.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:31,763 WARN  [FSHLog-0-hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8-prefix:0d751a622496,43029,1731847612705.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:31,763 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0d751a622496%2C43029%2C1731847612705.meta:.meta(num 1731847613728) roll requested
2024-11-17T12:47:31,763 INFO  [regionserver/0d751a622496:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C43029%2C1731847612705.meta.1731847651763.meta
2024-11-17T12:47:31,767 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/.tmp/info/47b5858012bb4107b8b90f7e2399776e is 1080, key is row0015/info:/1731847651670/Put/seqid=0
2024-11-17T12:47:31,769 WARN  [Thread-1041 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741897_1082
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:31,769 WARN  [Thread-1041 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741897_1082 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK], DatanodeInfoWithStorage[127.0.0.1:39105,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]) is bad.
2024-11-17T12:47:31,769 WARN  [Thread-1041 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741897_1082
2024-11-17T12:47:31,770 WARN  [Thread-1041 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]
2024-11-17T12:47:31,772 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:31,772 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:31,772 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:31,772 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:31,772 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:31,772 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847651763.meta
2024-11-17T12:47:31,773 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:31,773 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45021,DS-a138e678-6576-4789-8707-5ef24a4fc7ac,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:31,773 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
2024-11-17T12:47:31,773 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40685:40685),(127.0.0.1/127.0.0.1:34593:34593)]
2024-11-17T12:47:31,773 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta is not closed yet, will try archiving it next time
2024-11-17T12:47:31,774 WARN  [IPC Server handler 3 on default port 42079 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta has not been closed. Lease recovery is in progress. RecoveryId = 1084 for block blk_1073741834_1010
2024-11-17T12:47:31,774 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta after 1ms
2024-11-17T12:47:31,775 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741898_1083 (size=13586)
2024-11-17T12:47:31,776 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39105 is added to blk_1073741898_1083 (size=13586)
2024-11-17T12:47:31,776 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/.tmp/info/47b5858012bb4107b8b90f7e2399776e
2024-11-17T12:47:31,784 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/.tmp/info/47b5858012bb4107b8b90f7e2399776e as hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/47b5858012bb4107b8b90f7e2399776e
2024-11-17T12:47:31,792 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/47b5858012bb4107b8b90f7e2399776e, entries=8, sequenceid=77, filesize=13.3 K
2024-11-17T12:47:31,794 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=0 B/0 for dc2b8c9bdc7b25edc69ab039c0619f99 in 35ms, sequenceid=77, compaction requested=true
2024-11-17T12:47:31,797 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/b492cab2e0a74b6ab93a360f50888d77, hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/fc2a95a1cad1442ea020ad7e66fc262f, hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/41511714a5c241908c9d9462f096e81a, hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/d42ab62b652e4baf982503554f105fe8, hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/59540e2e4d74409da39ec8f13efea42a, hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/aeca021bcdf34a7ba78782f07258c715] to archive
2024-11-17T12:47:31,798 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/hbase/meta/1588230740/.tmp/info/4cc2c838ab9f4db9a0992c19e30b145a is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99./info:regioninfo/1731847614495/Put/seqid=0
2024-11-17T12:47:31,798 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-17T12:47:31,800 WARN  [Thread-1052 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741899_1085
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:31,800 WARN  [Thread-1052 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741899_1085 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK], DatanodeInfoWithStorage[127.0.0.1:39533,DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]) is bad.
2024-11-17T12:47:31,800 WARN  [Thread-1052 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741899_1085
2024-11-17T12:47:31,801 WARN  [Thread-1052 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]
2024-11-17T12:47:31,802 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/b492cab2e0a74b6ab93a360f50888d77 to hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/b492cab2e0a74b6ab93a360f50888d77
2024-11-17T12:47:31,804 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/fc2a95a1cad1442ea020ad7e66fc262f to hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/fc2a95a1cad1442ea020ad7e66fc262f
2024-11-17T12:47:31,807 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/41511714a5c241908c9d9462f096e81a to hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/41511714a5c241908c9d9462f096e81a
2024-11-17T12:47:31,810 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/d42ab62b652e4baf982503554f105fe8 to hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/d42ab62b652e4baf982503554f105fe8
2024-11-17T12:47:31,811 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/59540e2e4d74409da39ec8f13efea42a to hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/59540e2e4d74409da39ec8f13efea42a
2024-11-17T12:47:31,813 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/aeca021bcdf34a7ba78782f07258c715 to hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/info/aeca021bcdf34a7ba78782f07258c715
2024-11-17T12:47:31,814 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=0d751a622496:37969 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-11-17T12:47:31,814 WARN  [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [b492cab2e0a74b6ab93a360f50888d77=10347, fc2a95a1cad1442ea020ad7e66fc262f=12506, 41511714a5c241908c9d9462f096e81a=17994, d42ab62b652e4baf982503554f105fe8=6027, 59540e2e4d74409da39ec8f13efea42a=6027, aeca021bcdf34a7ba78782f07258c715=6027]
2024-11-17T12:47:31,821 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741900_1086 (size=7089)
2024-11-17T12:47:31,821 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39105 is added to blk_1073741900_1086 (size=7089)
2024-11-17T12:47:31,823 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/hbase/meta/1588230740/.tmp/info/4cc2c838ab9f4db9a0992c19e30b145a
2024-11-17T12:47:31,826 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/default/TestLogRolling-testLogRollOnDatanodeDeath/dc2b8c9bdc7b25edc69ab039c0619f99/recovered.edits/80.seqid, newMaxSeqId=80, maxSeqId=1
2024-11-17T12:47:31,827 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.
2024-11-17T12:47:31,827 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dc2b8c9bdc7b25edc69ab039c0619f99:
Waiting for close lock at 1731847651758Running coprocessor pre-close hooks at 1731847651758Disabling compacts and flushes for region at 1731847651758Disabling writes for close at 1731847651759 (+1 ms)Obtaining lock to block concurrent updates at 1731847651759Preparing flush snapshotting stores in dc2b8c9bdc7b25edc69ab039c0619f99 at 1731847651759Finished memstore snapshotting TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99., syncing WAL and waiting on mvcc, flushsize=dataSize=8608, getHeapSize=9456, getOffHeapSize=0, getCellsCount=8 at 1731847651759Flushing stores of TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99. at 1731847651762 (+3 ms)Flushing dc2b8c9bdc7b25edc69ab039c0619f99/info: creating writer at 1731847651762Flushing dc2b8c9bdc7b25edc69ab039c0619f99/info: appending metadata at 1731847651767 (+5 ms)Flushing dc2b8c9bdc7b25edc69ab039c0619f99/info: closing flushed file at 1731847651767Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c205b3: reopening flushed file at 1731847651783 (+16 ms)Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=0 B/0 for dc2b8c9bdc7b25edc69ab039c0619f99 in 35ms, sequenceid=77, compaction requested=true at 1731847651795 (+12 ms)Writing region close event to WAL at 1731847651820 (+25 ms)Running coprocessor post-close hooks at 1731847651827 (+7 ms)Closed at 1731847651827
2024-11-17T12:47:31,828 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731847614121.dc2b8c9bdc7b25edc69ab039c0619f99.
2024-11-17T12:47:31,854 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/hbase/meta/1588230740/.tmp/ns/121ef7f959c1495ebd35cb32706e93a3 is 43, key is default/ns:d/1731847613825/Put/seqid=0
2024-11-17T12:47:31,856 WARN  [Thread-1059 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741901_1087
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:31,857 WARN  [Thread-1059 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741901_1087 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK], DatanodeInfoWithStorage[127.0.0.1:39105,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]) is bad.
2024-11-17T12:47:31,857 WARN  [Thread-1059 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741901_1087
2024-11-17T12:47:31,858 WARN  [Thread-1059 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]
2024-11-17T12:47:31,868 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39105 is added to blk_1073741902_1088 (size=5153)
2024-11-17T12:47:31,868 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741902_1088 (size=5153)
2024-11-17T12:47:31,871 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/hbase/meta/1588230740/.tmp/ns/121ef7f959c1495ebd35cb32706e93a3
2024-11-17T12:47:31,894 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/hbase/meta/1588230740/.tmp/table/e00535c5cde74ae5bc1b5b520c57daa9 is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1731847614506/Put/seqid=0
2024-11-17T12:47:31,897 WARN  [Thread-1065 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741903_1089
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36351
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:31,897 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:48736 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741903_1089] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data4]'}, localName='127.0.0.1:39105', datanodeUuid='368a0fe1-98fb-4e11-a7ad-3861a4d4f891', xmitsInProgress=0}:Exception transferring block BP-1311131906-172.17.0.2-1731847610939:blk_1073741903_1089 to mirror 127.0.0.1:36351
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:31,897 WARN  [Thread-1065 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741903_1089 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39105,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK], DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]) is bad.
2024-11-17T12:47:31,897 WARN  [Thread-1065 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741903_1089
2024-11-17T12:47:31,897 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:48736 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741903_1089] {}] datanode.BlockReceiver(316): Block 1073741903 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-17T12:47:31,897 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-257700544_22 at /127.0.0.1:48736 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741903_1089] {}] datanode.DataXceiver(331): 127.0.0.1:39105:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48736 dst: /127.0.0.1:39105
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:31,898 WARN  [Thread-1065 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]
2024-11-17T12:47:31,903 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39105 is added to blk_1073741904_1090 (size=5424)
2024-11-17T12:47:31,903 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741904_1090 (size=5424)
2024-11-17T12:47:31,904 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/hbase/meta/1588230740/.tmp/table/e00535c5cde74ae5bc1b5b520c57daa9
2024-11-17T12:47:31,912 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/hbase/meta/1588230740/.tmp/info/4cc2c838ab9f4db9a0992c19e30b145a as hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/hbase/meta/1588230740/info/4cc2c838ab9f4db9a0992c19e30b145a
2024-11-17T12:47:31,921 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/hbase/meta/1588230740/info/4cc2c838ab9f4db9a0992c19e30b145a, entries=10, sequenceid=11, filesize=6.9 K
2024-11-17T12:47:31,922 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/hbase/meta/1588230740/.tmp/ns/121ef7f959c1495ebd35cb32706e93a3 as hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/hbase/meta/1588230740/ns/121ef7f959c1495ebd35cb32706e93a3
2024-11-17T12:47:31,930 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/hbase/meta/1588230740/ns/121ef7f959c1495ebd35cb32706e93a3, entries=2, sequenceid=11, filesize=5.0 K
2024-11-17T12:47:31,932 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/hbase/meta/1588230740/.tmp/table/e00535c5cde74ae5bc1b5b520c57daa9 as hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/hbase/meta/1588230740/table/e00535c5cde74ae5bc1b5b520c57daa9
2024-11-17T12:47:31,940 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/hbase/meta/1588230740/table/e00535c5cde74ae5bc1b5b520c57daa9, entries=2, sequenceid=11, filesize=5.3 K
2024-11-17T12:47:31,941 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 179ms, sequenceid=11, compaction requested=false
2024-11-17T12:47:31,951 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-17T12:47:31,952 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T12:47:31,952 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T12:47:31,952 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731847651762Running coprocessor pre-close hooks at 1731847651762Disabling compacts and flushes for region at 1731847651762Disabling writes for close at 1731847651762Obtaining lock to block concurrent updates at 1731847651762Preparing flush snapshotting stores in 1588230740 at 1731847651762Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1731847651763 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731847651774 (+11 ms)Flushing 1588230740/info: creating writer at 1731847651774Flushing 1588230740/info: appending metadata at 1731847651797 (+23 ms)Flushing 1588230740/info: closing flushed file at 1731847651797Flushing 1588230740/ns: creating writer at 1731847651831 (+34 ms)Flushing 1588230740/ns: appending metadata at 1731847651853 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1731847651853Flushing 1588230740/table: creating writer at 1731847651878 (+25 ms)Flushing 1588230740/table: appending metadata at 1731847651893 (+15 ms)Flushing 1588230740/table: closing flushed file at 1731847651893Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@743af329: reopening flushed file at 1731847651910 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43ef9b06: reopening flushed file at 1731847651921 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25db985c: reopening flushed file at 1731847651931 (+10 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 179ms, sequenceid=11, compaction requested=false at 1731847651941 (+10 ms)Writing region close event to WAL at 1731847651947 (+6 ms)Running coprocessor post-close hooks at 1731847651952 (+5 ms)Closed at 1731847651952
2024-11-17T12:47:31,953 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-17T12:47:31,962 INFO  [RS:0;0d751a622496:43029 {}] regionserver.HRegionServer(976): stopping server 0d751a622496,43029,1731847612705; all regions closed.
2024-11-17T12:47:31,962 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:31,963 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:31,963 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:31,963 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:31,963 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:31,965 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39105 is added to blk_1073741896_1081 (size=825)
2024-11-17T12:47:31,966 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741896_1081 (size=825)
2024-11-17T12:47:32,064 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.1731847635673 to hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/oldWALs/0d751a622496%2C43029%2C1731847612705.1731847635673
2024-11-17T12:47:32,091 INFO  [regionserver/0d751a622496:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T12:47:32,094 INFO  [regionserver/0d751a622496:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-17T12:47:32,094 INFO  [regionserver/0d751a622496:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-17T12:47:32,311 INFO  [regionserver/0d751a622496:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-17T12:47:32,311 INFO  [regionserver/0d751a622496:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-17T12:47:33,218 INFO  [regionserver/0d751a622496:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T12:47:33,869 INFO  [master/0d751a622496:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-17T12:47:33,869 INFO  [master/0d751a622496:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-17T12:47:34,377 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741825_1001 (size=7)
2024-11-17T12:47:34,380 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741836_1012 (size=76)
2024-11-17T12:47:34,432 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@2bea8b4b[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39533, datanodeUuid=6bb64b52-41b8-49af-bf13-ba9bcad2229b, infoPort=40685, infoSecurePort=0, ipcPort=41737, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939):Failed to transfer BP-1311131906-172.17.0.2-1731847610939:blk_1073741878_1061 to 127.0.0.1:36351 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:35,369 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@5346f772[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39105, datanodeUuid=368a0fe1-98fb-4e11-a7ad-3861a4d4f891, infoPort=34593, infoSecurePort=0, ipcPort=40369, storageInfo=lv=-57;cid=testClusterID;nsid=2099757610;c=1731847610939):Failed to transfer BP-1311131906-172.17.0.2-1731847610939:blk_1073741828_1004 to 127.0.0.1:36351 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:35,370 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741832_1008 (size=32)
2024-11-17T12:47:35,762 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224 after 4001ms
2024-11-17T12:47:35,775 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta after 4002ms
2024-11-17T12:47:36,760 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-17T12:47:36,762 DEBUG [RS:1;0d751a622496:37689 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/oldWALs
2024-11-17T12:47:36,762 INFO  [RS:1;0d751a622496:37689 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0d751a622496%2C37689%2C1731847613976:(num 1731847614224)
2024-11-17T12:47:36,763 DEBUG [RS:1;0d751a622496:37689 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:47:36,763 INFO  [RS:1;0d751a622496:37689 {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T12:47:36,763 INFO  [RS:1;0d751a622496:37689 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T12:47:36,763 INFO  [RS:1;0d751a622496:37689 {}] hbase.ChoreService(370): Chore service for: regionserver/0d751a622496:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-17T12:47:36,763 INFO  [RS:1;0d751a622496:37689 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-17T12:47:36,763 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T12:47:36,763 INFO  [RS:1;0d751a622496:37689 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-17T12:47:36,763 INFO  [RS:1;0d751a622496:37689 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-17T12:47:36,763 INFO  [RS:1;0d751a622496:37689 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T12:47:36,763 INFO  [RS:1;0d751a622496:37689 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37689
2024-11-17T12:47:36,768 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:36,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37689-0x1014a1efb3c0002, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0d751a622496,37689,1731847613976
2024-11-17T12:47:36,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T12:47:36,813 INFO  [RS:1;0d751a622496:37689 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T12:47:36,813 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0d751a622496,37689,1731847613976]
2024-11-17T12:47:36,828 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:47:36,831 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0d751a622496,37689,1731847613976 already deleted, retry=false
2024-11-17T12:47:36,831 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0d751a622496,37689,1731847613976 expired; onlineServers=1
2024-11-17T12:47:36,851 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:47:36,852 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:47:36,852 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:47:36,852 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:47:36,853 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:47:36,865 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:47:36,866 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:47:36,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37689-0x1014a1efb3c0002, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:47:36,923 INFO  [RS:1;0d751a622496:37689 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T12:47:36,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37689-0x1014a1efb3c0002, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:47:36,923 INFO  [RS:1;0d751a622496:37689 {}] regionserver.HRegionServer(1031): Exiting; stopping=0d751a622496,37689,1731847613976; zookeeper connection closed.
2024-11-17T12:47:36,923 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@334d1f0a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@334d1f0a
2024-11-17T12:47:36,963 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-17T12:47:36,967 DEBUG [RS:0;0d751a622496:43029 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/oldWALs
2024-11-17T12:47:36,967 INFO  [RS:0;0d751a622496:43029 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0d751a622496%2C43029%2C1731847612705.meta:.meta(num 1731847651763)
2024-11-17T12:47:36,968 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:36,968 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:36,968 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:36,968 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:36,968 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:36,970 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39105 is added to blk_1073741893_1077 (size=13514)
2024-11-17T12:47:36,970 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741893_1077 (size=13514)
2024-11-17T12:47:36,973 DEBUG [RS:0;0d751a622496:43029 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/oldWALs
2024-11-17T12:47:36,973 INFO  [RS:0;0d751a622496:43029 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0d751a622496%2C43029%2C1731847612705:(num 1731847651648)
2024-11-17T12:47:36,973 DEBUG [RS:0;0d751a622496:43029 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:47:36,973 INFO  [RS:0;0d751a622496:43029 {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T12:47:36,973 INFO  [RS:0;0d751a622496:43029 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T12:47:36,974 INFO  [RS:0;0d751a622496:43029 {}] hbase.ChoreService(370): Chore service for: regionserver/0d751a622496:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-17T12:47:36,974 INFO  [RS:0;0d751a622496:43029 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T12:47:36,974 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T12:47:36,974 INFO  [RS:0;0d751a622496:43029 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43029
2024-11-17T12:47:36,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T12:47:36,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43029-0x1014a1efb3c0001, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0d751a622496,43029,1731847612705
2024-11-17T12:47:36,997 INFO  [RS:0;0d751a622496:43029 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T12:47:36,998 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0d751a622496,43029,1731847612705]
2024-11-17T12:47:37,014 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0d751a622496,43029,1731847612705 already deleted, retry=false
2024-11-17T12:47:37,014 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0d751a622496,43029,1731847612705 expired; onlineServers=0
2024-11-17T12:47:37,014 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0d751a622496,37969,1731847612555' *****
2024-11-17T12:47:37,014 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-17T12:47:37,014 INFO  [M:0;0d751a622496:37969 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T12:47:37,015 INFO  [M:0;0d751a622496:37969 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T12:47:37,015 DEBUG [M:0;0d751a622496:37969 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-17T12:47:37,015 DEBUG [M:0;0d751a622496:37969 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-17T12:47:37,015 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-17T12:47:37,015 DEBUG [master/0d751a622496:0:becomeActiveMaster-HFileCleaner.large.0-1731847613062 {}] cleaner.HFileCleaner(306): Exit Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.large.0-1731847613062,5,FailOnTimeoutGroup]
2024-11-17T12:47:37,015 DEBUG [master/0d751a622496:0:becomeActiveMaster-HFileCleaner.small.0-1731847613063 {}] cleaner.HFileCleaner(306): Exit Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.small.0-1731847613063,5,FailOnTimeoutGroup]
2024-11-17T12:47:37,015 INFO  [M:0;0d751a622496:37969 {}] hbase.ChoreService(370): Chore service for: master/0d751a622496:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-17T12:47:37,016 INFO  [M:0;0d751a622496:37969 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T12:47:37,016 DEBUG [M:0;0d751a622496:37969 {}] master.HMaster(1795): Stopping service threads
2024-11-17T12:47:37,016 INFO  [M:0;0d751a622496:37969 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-17T12:47:37,016 INFO  [M:0;0d751a622496:37969 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T12:47:37,016 INFO  [M:0;0d751a622496:37969 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-17T12:47:37,016 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-17T12:47:37,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-17T12:47:37,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:47:37,023 DEBUG [M:0;0d751a622496:37969 {}] zookeeper.ZKUtil(347): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-17T12:47:37,023 WARN  [M:0;0d751a622496:37969 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-17T12:47:37,023 INFO  [M:0;0d751a622496:37969 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/.lastflushedseqids
2024-11-17T12:47:37,031 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39105 is added to blk_1073741905_1091 (size=130)
2024-11-17T12:47:37,031 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741905_1091 (size=130)
2024-11-17T12:47:37,031 INFO  [M:0;0d751a622496:37969 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-17T12:47:37,032 INFO  [M:0;0d751a622496:37969 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-17T12:47:37,032 DEBUG [M:0;0d751a622496:37969 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T12:47:37,032 INFO  [M:0;0d751a622496:37969 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:47:37,032 DEBUG [M:0;0d751a622496:37969 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:47:37,032 DEBUG [M:0;0d751a622496:37969 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T12:47:37,032 DEBUG [M:0;0d751a622496:37969 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:47:37,032 INFO  [M:0;0d751a622496:37969 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.25 KB heapSize=29.49 KB
2024-11-17T12:47:37,053 DEBUG [M:0;0d751a622496:37969 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/633e7fd0001247fc90184681f5ad8116 is 82, key is hbase:meta,,1/info:regioninfo/1731847613770/Put/seqid=0
2024-11-17T12:47:37,055 WARN  [Thread-1083 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741906_1092
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:37,055 WARN  [Thread-1083 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741906_1092 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK], DatanodeInfoWithStorage[127.0.0.1:39533,DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]) is bad.
2024-11-17T12:47:37,055 WARN  [Thread-1083 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741906_1092
2024-11-17T12:47:37,056 WARN  [Thread-1083 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]
2024-11-17T12:47:37,061 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741907_1093 (size=5672)
2024-11-17T12:47:37,061 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39105 is added to blk_1073741907_1093 (size=5672)
2024-11-17T12:47:37,061 INFO  [M:0;0d751a622496:37969 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/633e7fd0001247fc90184681f5ad8116
2024-11-17T12:47:37,087 DEBUG [M:0;0d751a622496:37969 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3e3cc05f4fda4096892d2c34ba138ddc is 774, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731847614511/Put/seqid=0
2024-11-17T12:47:37,088 WARN  [Thread-1090 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741908_1094
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:37,089 WARN  [Thread-1090 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741908_1094 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK], DatanodeInfoWithStorage[127.0.0.1:39105,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]) is bad.
2024-11-17T12:47:37,089 WARN  [Thread-1090 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741908_1094
2024-11-17T12:47:37,089 WARN  [Thread-1090 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]
2024-11-17T12:47:37,101 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741909_1095 (size=6255)
2024-11-17T12:47:37,102 INFO  [M:0;0d751a622496:37969 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.58 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3e3cc05f4fda4096892d2c34ba138ddc
2024-11-17T12:47:37,102 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39105 is added to blk_1073741909_1095 (size=6255)
2024-11-17T12:47:37,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43029-0x1014a1efb3c0001, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:47:37,106 INFO  [RS:0;0d751a622496:43029 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T12:47:37,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43029-0x1014a1efb3c0001, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:47:37,106 INFO  [RS:0;0d751a622496:43029 {}] regionserver.HRegionServer(1031): Exiting; stopping=0d751a622496,43029,1731847612705; zookeeper connection closed.
2024-11-17T12:47:37,106 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@28e31923 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@28e31923
2024-11-17T12:47:37,106 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete
2024-11-17T12:47:37,109 INFO  [M:0;0d751a622496:37969 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3e3cc05f4fda4096892d2c34ba138ddc
2024-11-17T12:47:37,128 DEBUG [M:0;0d751a622496:37969 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b91ddd420bc7413f82194c5111b14d60 is 69, key is 0d751a622496,37689,1731847613976/rs:state/1731847614067/Put/seqid=0
2024-11-17T12:47:37,130 WARN  [Thread-1097 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741910_1096
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36351
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:37,130 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_475526744_22 at /127.0.0.1:33462 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741910_1096] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data8]'}, localName='127.0.0.1:39533', datanodeUuid='6bb64b52-41b8-49af-bf13-ba9bcad2229b', xmitsInProgress=0}:Exception transferring block BP-1311131906-172.17.0.2-1731847610939:blk_1073741910_1096 to mirror 127.0.0.1:36351
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:37,130 WARN  [Thread-1097 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741910_1096 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39533,DS-f3d38202-38ef-4546-83b8-c8dfc2e7045b,DISK], DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]) is bad.
2024-11-17T12:47:37,130 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_475526744_22 at /127.0.0.1:33462 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741910_1096] {}] datanode.BlockReceiver(316): Block 1073741910 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-17T12:47:37,130 WARN  [Thread-1097 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741910_1096
2024-11-17T12:47:37,131 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_475526744_22 at /127.0.0.1:33462 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741910_1096] {}] datanode.DataXceiver(331): 127.0.0.1:39533:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:33462 dst: /127.0.0.1:39533
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:37,131 WARN  [Thread-1097 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]
2024-11-17T12:47:37,135 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741911_1097 (size=5224)
2024-11-17T12:47:37,135 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39105 is added to blk_1073741911_1097 (size=5224)
2024-11-17T12:47:37,136 INFO  [M:0;0d751a622496:37969 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b91ddd420bc7413f82194c5111b14d60
2024-11-17T12:47:37,158 DEBUG [M:0;0d751a622496:37969 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a6f3d267840046dbb12fc93474ce0ce3 is 52, key is load_balancer_on/state:d/1731847613955/Put/seqid=0
2024-11-17T12:47:37,171 WARN  [Thread-1104 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741912_1098
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:36351
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:37,171 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_475526744_22 at /127.0.0.1:48814 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741912_1098] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data4]'}, localName='127.0.0.1:39105', datanodeUuid='368a0fe1-98fb-4e11-a7ad-3861a4d4f891', xmitsInProgress=0}:Exception transferring block BP-1311131906-172.17.0.2-1731847610939:blk_1073741912_1098 to mirror 127.0.0.1:36351
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:37,172 WARN  [Thread-1104 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1311131906-172.17.0.2-1731847610939:blk_1073741912_1098 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39105,DS-6c3a8097-026a-47a3-95f5-d5550b7d9a0b,DISK], DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]) is bad.
2024-11-17T12:47:37,172 WARN  [Thread-1104 {}] hdfs.DataStreamer(1850): Abandoning BP-1311131906-172.17.0.2-1731847610939:blk_1073741912_1098
2024-11-17T12:47:37,172 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_475526744_22 at /127.0.0.1:48814 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741912_1098] {}] datanode.BlockReceiver(316): Block 1073741912 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-17T12:47:37,172 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_475526744_22 at /127.0.0.1:48814 [Receiving block BP-1311131906-172.17.0.2-1731847610939:blk_1073741912_1098] {}] datanode.DataXceiver(331): 127.0.0.1:39105:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48814 dst: /127.0.0.1:39105
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:37,172 WARN  [Thread-1104 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:36351,DS-a1073542-bba2-4b3f-8007-c55f080d1c21,DISK]
2024-11-17T12:47:37,187 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741913_1099 (size=5056)
2024-11-17T12:47:37,187 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39105 is added to blk_1073741913_1099 (size=5056)
2024-11-17T12:47:37,188 INFO  [M:0;0d751a622496:37969 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a6f3d267840046dbb12fc93474ce0ce3
2024-11-17T12:47:37,196 DEBUG [M:0;0d751a622496:37969 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/633e7fd0001247fc90184681f5ad8116 as hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/633e7fd0001247fc90184681f5ad8116
2024-11-17T12:47:37,202 INFO  [M:0;0d751a622496:37969 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/633e7fd0001247fc90184681f5ad8116, entries=8, sequenceid=60, filesize=5.5 K
2024-11-17T12:47:37,204 DEBUG [M:0;0d751a622496:37969 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3e3cc05f4fda4096892d2c34ba138ddc as hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3e3cc05f4fda4096892d2c34ba138ddc
2024-11-17T12:47:37,210 INFO  [M:0;0d751a622496:37969 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3e3cc05f4fda4096892d2c34ba138ddc
2024-11-17T12:47:37,210 INFO  [M:0;0d751a622496:37969 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3e3cc05f4fda4096892d2c34ba138ddc, entries=6, sequenceid=60, filesize=6.1 K
2024-11-17T12:47:37,212 DEBUG [M:0;0d751a622496:37969 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b91ddd420bc7413f82194c5111b14d60 as hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b91ddd420bc7413f82194c5111b14d60
2024-11-17T12:47:37,220 INFO  [M:0;0d751a622496:37969 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b91ddd420bc7413f82194c5111b14d60, entries=2, sequenceid=60, filesize=5.1 K
2024-11-17T12:47:37,221 DEBUG [M:0;0d751a622496:37969 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a6f3d267840046dbb12fc93474ce0ce3 as hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/a6f3d267840046dbb12fc93474ce0ce3
2024-11-17T12:47:37,229 INFO  [M:0;0d751a622496:37969 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/a6f3d267840046dbb12fc93474ce0ce3, entries=1, sequenceid=60, filesize=4.9 K
2024-11-17T12:47:37,230 INFO  [M:0;0d751a622496:37969 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 198ms, sequenceid=60, compaction requested=false
2024-11-17T12:47:37,232 INFO  [M:0;0d751a622496:37969 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:47:37,232 DEBUG [M:0;0d751a622496:37969 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731847657032Disabling compacts and flushes for region at 1731847657032Disabling writes for close at 1731847657032Obtaining lock to block concurrent updates at 1731847657032Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731847657032Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23805, getHeapSize=30136, getOffHeapSize=0, getCellsCount=71 at 1731847657033 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731847657033Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731847657034 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731847657052 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731847657052Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731847657068 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731847657086 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731847657086Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731847657109 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731847657127 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731847657127Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731847657141 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731847657157 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731847657157Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@91d6a31: reopening flushed file at 1731847657195 (+38 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e84db03: reopening flushed file at 1731847657203 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f4b766b: reopening flushed file at 1731847657211 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45daa5f6: reopening flushed file at 1731847657220 (+9 ms)Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 198ms, sequenceid=60, compaction requested=false at 1731847657230 (+10 ms)Writing region close event to WAL at 1731847657232 (+2 ms)Closed at 1731847657232
2024-11-17T12:47:37,233 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:37,233 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:37,233 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:37,233 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:37,233 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:37,236 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39105 is added to blk_1073741891_1074 (size=1045)
2024-11-17T12:47:37,236 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39533 is added to blk_1073741891_1074 (size=1045)
2024-11-17T12:47:37,368 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-17T12:47:37,394 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:47:37,395 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:47:37,395 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:47:37,396 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:47:37,396 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:47:37,397 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:47:37,403 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:47:37,405 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:47:37,770 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:37,776 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:38,389 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@50864177 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1311131906-172.17.0.2-1731847610939:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:45021,null,null])
java.net.ConnectException: Call From 0d751a622496/172.17.0.2 to localhost:33121 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-17T12:47:38,770 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:38,776 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:39,093 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/WALs/0d751a622496,37969,1731847612555/0d751a622496%2C37969%2C1731847612555.1731847612841 to hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/oldWALs/0d751a622496%2C37969%2C1731847612555.1731847612841
2024-11-17T12:47:39,097 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/MasterData/oldWALs/0d751a622496%2C37969%2C1731847612555.1731847612841 to hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/oldWALs/0d751a622496%2C37969%2C1731847612555.1731847612841$masterlocalwal$
2024-11-17T12:47:39,098 INFO  [M:0;0d751a622496:37969 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-17T12:47:39,098 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T12:47:39,098 INFO  [M:0;0d751a622496:37969 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37969
2024-11-17T12:47:39,098 INFO  [M:0;0d751a622496:37969 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T12:47:39,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:47:39,246 INFO  [M:0;0d751a622496:37969 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T12:47:39,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37969-0x1014a1efb3c0000, quorum=127.0.0.1:62721, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:47:39,287 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@30d4c51a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:47:39,288 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34827478{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:47:39,288 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:47:39,288 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3edaa0c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:47:39,288 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@796aacdc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/hadoop.log.dir/,STOPPED}
2024-11-17T12:47:39,290 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T12:47:39,290 WARN  [BP-1311131906-172.17.0.2-1731847610939 heartbeating to localhost/127.0.0.1:42079 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T12:47:39,290 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T12:47:39,290 WARN  [BP-1311131906-172.17.0.2-1731847610939 heartbeating to localhost/127.0.0.1:42079 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1311131906-172.17.0.2-1731847610939 (Datanode Uuid 368a0fe1-98fb-4e11-a7ad-3861a4d4f891) service to localhost/127.0.0.1:42079
2024-11-17T12:47:39,290 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@31a99994 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1311131906-172.17.0.2-1731847610939:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:45021,null,null])
java.io.InterruptedIOException: DestHost:destPort localhost:33121 , LocalHost:localPort 0d751a622496/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
Caused by: java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-17T12:47:39,291 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@31a99994 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1311131906-172.17.0.2-1731847610939:blk_1073741837_1013; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:39105,null,null], DatanodeInfoWithStorage[127.0.0.1:45021,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: No block pool offer service for bpid=BP-1311131906-172.17.0.2-1731847610939
2024-11-17T12:47:39,291 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data3/current/BP-1311131906-172.17.0.2-1731847610939 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:47:39,291 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@31a99994 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1311131906-172.17.0.2-1731847610939:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:45021,null,null])
java.io.IOException: No block pool offer service for bpid=BP-1311131906-172.17.0.2-1731847610939
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:39,291 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@31a99994 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1311131906-172.17.0.2-1731847610939:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:39105,null,null])
java.io.IOException: No block pool offer service for bpid=BP-1311131906-172.17.0.2-1731847610939
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:39,292 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data4/current/BP-1311131906-172.17.0.2-1731847610939 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:47:39,292 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@31a99994 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1311131906-172.17.0.2-1731847610939:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:45021,null,null], DatanodeInfoWithStorage[127.0.0.1:39105,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-1311131906-172.17.0.2-1731847610939:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:45021,null,null], DatanodeInfoWithStorage[127.0.0.1:39105,null,null]]
2024-11-17T12:47:39,292 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T12:47:39,294 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4b9d70b4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:47:39,294 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@47432b7b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:47:39,294 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:47:39,294 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@463ced82{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:47:39,294 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@574823ec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/hadoop.log.dir/,STOPPED}
2024-11-17T12:47:39,296 WARN  [BP-1311131906-172.17.0.2-1731847610939 heartbeating to localhost/127.0.0.1:42079 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T12:47:39,296 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T12:47:39,296 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T12:47:39,296 WARN  [BP-1311131906-172.17.0.2-1731847610939 heartbeating to localhost/127.0.0.1:42079 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1311131906-172.17.0.2-1731847610939 (Datanode Uuid 6bb64b52-41b8-49af-bf13-ba9bcad2229b) service to localhost/127.0.0.1:42079
2024-11-17T12:47:39,296 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data7/current/BP-1311131906-172.17.0.2-1731847610939 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:47:39,297 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/cluster_b466ef79-1409-59eb-0683-e06e3e5cf676/data/data8/current/BP-1311131906-172.17.0.2-1731847610939 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:47:39,297 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T12:47:39,303 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2295376c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T12:47:39,303 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@54adbc26{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:47:39,304 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:47:39,304 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4ac253d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:47:39,304 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ab5393f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/hadoop.log.dir/,STOPPED}
2024-11-17T12:47:39,313 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-17T12:47:39,348 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-17T12:47:39,356 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=154 (was 78)
Potentially hanging thread: nioEventLoopGroup-20-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:42079 from jenkins.hfs.2
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: ForkJoinPool-2-worker-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:42079 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:44873
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:44873
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$899/0x00007f52e0befdc8.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42079
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42079
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$899/0x00007f52e0befdc8.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:42079
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:42079
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'DataNode' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: HMaster-EventLoopGroup-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:42079 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-16-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42079
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42079
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:42079
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:42079 from jenkins.hfs.3
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=450 (was 404) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=197 (was 205), ProcessCount=11 (was 11), AvailableMemoryMB=5557 (was 6718)
2024-11-17T12:47:39,363 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=154, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=197, ProcessCount=11, AvailableMemoryMB=5557
2024-11-17T12:47:39,363 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-17T12:47:39,363 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/hadoop.log.dir so I do NOT create it in target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5
2024-11-17T12:47:39,363 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/11f3bcd0-90e4-ea3c-c5f8-b1c5d4370361/hadoop.tmp.dir so I do NOT create it in target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5
2024-11-17T12:47:39,363 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/cluster_b93fc69f-3ebf-86a6-e261-c77f3932f38f, deleteOnExit=true
2024-11-17T12:47:39,363 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-17T12:47:39,364 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/test.cache.data in system properties and HBase conf
2024-11-17T12:47:39,364 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/hadoop.tmp.dir in system properties and HBase conf
2024-11-17T12:47:39,364 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/hadoop.log.dir in system properties and HBase conf
2024-11-17T12:47:39,364 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-17T12:47:39,364 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-17T12:47:39,364 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-17T12:47:39,364 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-17T12:47:39,364 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-17T12:47:39,365 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-17T12:47:39,365 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-17T12:47:39,365 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T12:47:39,365 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-17T12:47:39,365 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-17T12:47:39,365 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T12:47:39,365 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T12:47:39,365 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-17T12:47:39,365 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/nfs.dump.dir in system properties and HBase conf
2024-11-17T12:47:39,365 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/java.io.tmpdir in system properties and HBase conf
2024-11-17T12:47:39,365 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T12:47:39,365 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-17T12:47:39,366 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-17T12:47:39,379 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T12:47:39,682 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:47:39,688 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:47:39,703 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:47:39,703 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:47:39,703 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T12:47:39,704 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:47:39,704 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@49d720be{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:47:39,705 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@33392a77{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:47:39,771 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:39,777 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:39,819 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6a724f0c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/java.io.tmpdir/jetty-localhost-45367-hadoop-hdfs-3_4_1-tests_jar-_-any-9970945690454039141/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T12:47:39,820 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@17be9294{HTTP/1.1, (http/1.1)}{localhost:45367}
2024-11-17T12:47:39,820 INFO  [Time-limited test {}] server.Server(415): Started @155062ms
2024-11-17T12:47:39,836 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T12:47:40,107 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:47:40,111 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:47:40,112 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:47:40,112 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:47:40,112 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-17T12:47:40,113 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@78725bd6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:47:40,113 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@78745d04{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:47:40,221 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1b91ed3f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/java.io.tmpdir/jetty-localhost-37171-hadoop-hdfs-3_4_1-tests_jar-_-any-1113399875490679/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:47:40,221 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@16ec9c96{HTTP/1.1, (http/1.1)}{localhost:37171}
2024-11-17T12:47:40,222 INFO  [Time-limited test {}] server.Server(415): Started @155463ms
2024-11-17T12:47:40,223 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T12:47:40,250 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:47:40,254 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:47:40,255 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:47:40,255 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:47:40,255 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T12:47:40,256 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70644e5e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:47:40,256 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@f09ac85{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:47:40,360 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1a8d3987{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/java.io.tmpdir/jetty-localhost-46149-hadoop-hdfs-3_4_1-tests_jar-_-any-17842680383465566840/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:47:40,360 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3e58ccd6{HTTP/1.1, (http/1.1)}{localhost:46149}
2024-11-17T12:47:40,360 INFO  [Time-limited test {}] server.Server(415): Started @155602ms
2024-11-17T12:47:40,362 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T12:47:40,479 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-17T12:47:40,480 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T12:47:40,480 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-17T12:47:40,480 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-17T12:47:40,772 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:40,778 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:40,933 WARN  [Thread-1201 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/cluster_b93fc69f-3ebf-86a6-e261-c77f3932f38f/data/data1/current/BP-1897123262-172.17.0.2-1731847659391/current, will proceed with Du for space computation calculation, 
2024-11-17T12:47:40,933 WARN  [Thread-1202 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/cluster_b93fc69f-3ebf-86a6-e261-c77f3932f38f/data/data2/current/BP-1897123262-172.17.0.2-1731847659391/current, will proceed with Du for space computation calculation, 
2024-11-17T12:47:40,954 WARN  [Thread-1165 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T12:47:40,956 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5d5f85c0f94bf252 with lease ID 0xd0c3c5f11ba5aa5a: Processing first storage report for DS-40ff578d-b4b3-4910-8233-f15f8ec9c022 from datanode DatanodeRegistration(127.0.0.1:36187, datanodeUuid=75852f8c-5eae-4c94-85d9-124ca4775ed8, infoPort=44187, infoSecurePort=0, ipcPort=43631, storageInfo=lv=-57;cid=testClusterID;nsid=1105269691;c=1731847659391)
2024-11-17T12:47:40,956 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5d5f85c0f94bf252 with lease ID 0xd0c3c5f11ba5aa5a: from storage DS-40ff578d-b4b3-4910-8233-f15f8ec9c022 node DatanodeRegistration(127.0.0.1:36187, datanodeUuid=75852f8c-5eae-4c94-85d9-124ca4775ed8, infoPort=44187, infoSecurePort=0, ipcPort=43631, storageInfo=lv=-57;cid=testClusterID;nsid=1105269691;c=1731847659391), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:47:40,956 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5d5f85c0f94bf252 with lease ID 0xd0c3c5f11ba5aa5a: Processing first storage report for DS-dcb3d0c4-a239-4bff-83f0-5aec39a96b0a from datanode DatanodeRegistration(127.0.0.1:36187, datanodeUuid=75852f8c-5eae-4c94-85d9-124ca4775ed8, infoPort=44187, infoSecurePort=0, ipcPort=43631, storageInfo=lv=-57;cid=testClusterID;nsid=1105269691;c=1731847659391)
2024-11-17T12:47:40,956 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5d5f85c0f94bf252 with lease ID 0xd0c3c5f11ba5aa5a: from storage DS-dcb3d0c4-a239-4bff-83f0-5aec39a96b0a node DatanodeRegistration(127.0.0.1:36187, datanodeUuid=75852f8c-5eae-4c94-85d9-124ca4775ed8, infoPort=44187, infoSecurePort=0, ipcPort=43631, storageInfo=lv=-57;cid=testClusterID;nsid=1105269691;c=1731847659391), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:47:41,098 WARN  [Thread-1212 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/cluster_b93fc69f-3ebf-86a6-e261-c77f3932f38f/data/data3/current/BP-1897123262-172.17.0.2-1731847659391/current, will proceed with Du for space computation calculation, 
2024-11-17T12:47:41,098 WARN  [Thread-1213 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/cluster_b93fc69f-3ebf-86a6-e261-c77f3932f38f/data/data4/current/BP-1897123262-172.17.0.2-1731847659391/current, will proceed with Du for space computation calculation, 
2024-11-17T12:47:41,120 WARN  [Thread-1188 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T12:47:41,122 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9088a28a9bed6ccc with lease ID 0xd0c3c5f11ba5aa5b: Processing first storage report for DS-c108bbc1-7b60-49b7-be4e-ffef61fc3522 from datanode DatanodeRegistration(127.0.0.1:34729, datanodeUuid=4d688259-a8bb-437b-95bd-a86e6fe826d7, infoPort=35693, infoSecurePort=0, ipcPort=38319, storageInfo=lv=-57;cid=testClusterID;nsid=1105269691;c=1731847659391)
2024-11-17T12:47:41,123 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9088a28a9bed6ccc with lease ID 0xd0c3c5f11ba5aa5b: from storage DS-c108bbc1-7b60-49b7-be4e-ffef61fc3522 node DatanodeRegistration(127.0.0.1:34729, datanodeUuid=4d688259-a8bb-437b-95bd-a86e6fe826d7, infoPort=35693, infoSecurePort=0, ipcPort=38319, storageInfo=lv=-57;cid=testClusterID;nsid=1105269691;c=1731847659391), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:47:41,123 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9088a28a9bed6ccc with lease ID 0xd0c3c5f11ba5aa5b: Processing first storage report for DS-b03802a7-7ed7-4def-ae03-0ad8cd494bc0 from datanode DatanodeRegistration(127.0.0.1:34729, datanodeUuid=4d688259-a8bb-437b-95bd-a86e6fe826d7, infoPort=35693, infoSecurePort=0, ipcPort=38319, storageInfo=lv=-57;cid=testClusterID;nsid=1105269691;c=1731847659391)
2024-11-17T12:47:41,123 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9088a28a9bed6ccc with lease ID 0xd0c3c5f11ba5aa5b: from storage DS-b03802a7-7ed7-4def-ae03-0ad8cd494bc0 node DatanodeRegistration(127.0.0.1:34729, datanodeUuid=4d688259-a8bb-437b-95bd-a86e6fe826d7, infoPort=35693, infoSecurePort=0, ipcPort=38319, storageInfo=lv=-57;cid=testClusterID;nsid=1105269691;c=1731847659391), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:47:41,195 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5
2024-11-17T12:47:41,213 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/cluster_b93fc69f-3ebf-86a6-e261-c77f3932f38f/zookeeper_0, clientPort=60904, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/cluster_b93fc69f-3ebf-86a6-e261-c77f3932f38f/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/cluster_b93fc69f-3ebf-86a6-e261-c77f3932f38f/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-17T12:47:41,214 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60904
2024-11-17T12:47:41,215 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:47:41,216 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:47:41,226 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34729 is added to blk_1073741825_1001 (size=7)
2024-11-17T12:47:41,227 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36187 is added to blk_1073741825_1001 (size=7)
2024-11-17T12:47:41,228 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb with version=8
2024-11-17T12:47:41,228 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/hbase-staging
2024-11-17T12:47:41,231 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/0d751a622496:0 server-side Connection retries=45
2024-11-17T12:47:41,231 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:47:41,231 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T12:47:41,231 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T12:47:41,231 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:47:41,231 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T12:47:41,231 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-17T12:47:41,231 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T12:47:41,232 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45911
2024-11-17T12:47:41,234 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45911 connecting to ZooKeeper ensemble=127.0.0.1:60904
2024-11-17T12:47:41,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:459110x0, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T12:47:41,279 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45911-0x1014a1fb9610000 connected
2024-11-17T12:47:41,349 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:47:41,351 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:47:41,353 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:47:41,354 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb, hbase.cluster.distributed=false
2024-11-17T12:47:41,355 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T12:47:41,356 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45911
2024-11-17T12:47:41,356 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45911
2024-11-17T12:47:41,356 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45911
2024-11-17T12:47:41,357 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45911
2024-11-17T12:47:41,357 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45911
2024-11-17T12:47:41,375 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/0d751a622496:0 server-side Connection retries=45
2024-11-17T12:47:41,375 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:47:41,376 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T12:47:41,376 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T12:47:41,376 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:47:41,376 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T12:47:41,376 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-17T12:47:41,376 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T12:47:41,377 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34727
2024-11-17T12:47:41,378 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34727 connecting to ZooKeeper ensemble=127.0.0.1:60904
2024-11-17T12:47:41,379 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:47:41,381 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:47:41,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:347270x0, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T12:47:41,391 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:347270x0, quorum=127.0.0.1:60904, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:47:41,391 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-17T12:47:41,391 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34727-0x1014a1fb9610001 connected
2024-11-17T12:47:41,392 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-17T12:47:41,392 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34727-0x1014a1fb9610001, quorum=127.0.0.1:60904, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-17T12:47:41,393 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34727-0x1014a1fb9610001, quorum=127.0.0.1:60904, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T12:47:41,394 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34727
2024-11-17T12:47:41,394 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34727
2024-11-17T12:47:41,395 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34727
2024-11-17T12:47:41,395 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34727
2024-11-17T12:47:41,395 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34727
2024-11-17T12:47:41,409 DEBUG [M:0;0d751a622496:45911 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0d751a622496:45911
2024-11-17T12:47:41,409 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0d751a622496,45911,1731847661230
2024-11-17T12:47:41,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:47:41,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34727-0x1014a1fb9610001, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:47:41,421 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0d751a622496,45911,1731847661230
2024-11-17T12:47:41,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:47:41,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34727-0x1014a1fb9610001, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-17T12:47:41,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34727-0x1014a1fb9610001, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:47:41,433 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-17T12:47:41,433 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0d751a622496,45911,1731847661230 from backup master directory
2024-11-17T12:47:41,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0d751a622496,45911,1731847661230
2024-11-17T12:47:41,440 WARN  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T12:47:41,440 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0d751a622496,45911,1731847661230
2024-11-17T12:47:41,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34727-0x1014a1fb9610001, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:47:41,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:47:41,446 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/hbase.id] with ID: 46cb9b08-f5f9-4bd0-a740-c11862549a67
2024-11-17T12:47:41,446 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/.tmp/hbase.id
2024-11-17T12:47:41,453 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34729 is added to blk_1073741826_1002 (size=42)
2024-11-17T12:47:41,454 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36187 is added to blk_1073741826_1002 (size=42)
2024-11-17T12:47:41,454 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/.tmp/hbase.id]:[hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/hbase.id]
2024-11-17T12:47:41,468 INFO  [master/0d751a622496:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:47:41,468 INFO  [master/0d751a622496:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-17T12:47:41,470 INFO  [master/0d751a622496:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-17T12:47:41,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34727-0x1014a1fb9610001, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:47:41,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:47:41,485 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34729 is added to blk_1073741827_1003 (size=196)
2024-11-17T12:47:41,486 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36187 is added to blk_1073741827_1003 (size=196)
2024-11-17T12:47:41,773 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:41,778 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:41,887 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-17T12:47:41,889 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-17T12:47:41,889 INFO  [master/0d751a622496:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T12:47:41,898 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34729 is added to blk_1073741828_1004 (size=1189)
2024-11-17T12:47:41,898 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36187 is added to blk_1073741828_1004 (size=1189)
2024-11-17T12:47:41,899 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/data/master/store
2024-11-17T12:47:41,909 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36187 is added to blk_1073741829_1005 (size=34)
2024-11-17T12:47:41,909 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34729 is added to blk_1073741829_1005 (size=34)
2024-11-17T12:47:41,910 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:47:41,910 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T12:47:41,910 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:47:41,910 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:47:41,910 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T12:47:41,910 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:47:41,910 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:47:41,910 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731847661910Disabling compacts and flushes for region at 1731847661910Disabling writes for close at 1731847661910Writing region close event to WAL at 1731847661910Closed at 1731847661910
2024-11-17T12:47:41,911 WARN  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/data/master/store/.initializing
2024-11-17T12:47:41,911 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/WALs/0d751a622496,45911,1731847661230
2024-11-17T12:47:41,914 INFO  [master/0d751a622496:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C45911%2C1731847661230, suffix=, logDir=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/WALs/0d751a622496,45911,1731847661230, archiveDir=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/oldWALs, maxLogs=10
2024-11-17T12:47:41,914 INFO  [master/0d751a622496:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C45911%2C1731847661230.1731847661914
2024-11-17T12:47:41,919 INFO  [master/0d751a622496:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/WALs/0d751a622496,45911,1731847661230/0d751a622496%2C45911%2C1731847661230.1731847661914
2024-11-17T12:47:41,933 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35693:35693),(127.0.0.1/127.0.0.1:44187:44187)]
2024-11-17T12:47:41,939 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-17T12:47:41,939 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:47:41,939 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:47:41,939 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:47:41,941 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:47:41,942 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-17T12:47:41,942 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:47:41,942 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:47:41,943 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:47:41,944 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-17T12:47:41,944 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:47:41,944 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:47:41,945 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:47:41,946 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-17T12:47:41,946 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:47:41,947 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:47:41,947 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:47:41,948 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-17T12:47:41,948 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:47:41,949 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:47:41,949 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:47:41,950 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:47:41,950 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:47:41,952 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:47:41,952 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:47:41,952 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-17T12:47:41,953 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:47:41,955 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T12:47:41,956 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=811595, jitterRate=0.03199674189090729}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-17T12:47:41,957 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731847661939Initializing all the Stores at 1731847661940 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847661940Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847661940Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847661940Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847661940Cleaning up temporary data from old regions at 1731847661952 (+12 ms)Region opened successfully at 1731847661957 (+5 ms)
2024-11-17T12:47:41,957 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-17T12:47:41,960 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5746aaa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0d751a622496/172.17.0.2:0
2024-11-17T12:47:41,961 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-17T12:47:41,961 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-17T12:47:41,961 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-17T12:47:41,962 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-17T12:47:41,962 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-17T12:47:41,962 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-17T12:47:41,963 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-17T12:47:41,965 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-17T12:47:41,965 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-17T12:47:42,003 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-17T12:47:42,004 INFO  [master/0d751a622496:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-17T12:47:42,005 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-17T12:47:42,015 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-17T12:47:42,015 INFO  [master/0d751a622496:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-17T12:47:42,017 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-17T12:47:42,023 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-17T12:47:42,026 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-17T12:47:42,037 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-17T12:47:42,040 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-17T12:47:42,048 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-17T12:47:42,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T12:47:42,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:47:42,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34727-0x1014a1fb9610001, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T12:47:42,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34727-0x1014a1fb9610001, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:47:42,057 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0d751a622496,45911,1731847661230, sessionid=0x1014a1fb9610000, setting cluster-up flag (Was=false)
2024-11-17T12:47:42,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:47:42,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34727-0x1014a1fb9610001, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:47:42,098 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-17T12:47:42,099 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0d751a622496,45911,1731847661230
2024-11-17T12:47:42,115 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34727-0x1014a1fb9610001, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:47:42,115 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:47:42,140 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-17T12:47:42,142 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0d751a622496,45911,1731847661230
2024-11-17T12:47:42,143 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-17T12:47:42,145 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-17T12:47:42,145 INFO  [master/0d751a622496:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-17T12:47:42,145 INFO  [master/0d751a622496:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-17T12:47:42,146 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0d751a622496,45911,1731847661230
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-17T12:47:42,147 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:47:42,147 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:47:42,147 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:47:42,147 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:47:42,147 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0d751a622496:0, corePoolSize=10, maxPoolSize=10
2024-11-17T12:47:42,148 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:47:42,148 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0d751a622496:0, corePoolSize=2, maxPoolSize=2
2024-11-17T12:47:42,148 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:47:42,149 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731847692149
2024-11-17T12:47:42,149 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-17T12:47:42,149 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-17T12:47:42,149 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-17T12:47:42,149 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-17T12:47:42,149 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-17T12:47:42,149 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-17T12:47:42,149 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T12:47:42,150 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T12:47:42,150 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-17T12:47:42,150 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-17T12:47:42,150 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-17T12:47:42,150 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-17T12:47:42,150 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-17T12:47:42,150 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-17T12:47:42,151 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.large.0-1731847662150,5,FailOnTimeoutGroup]
2024-11-17T12:47:42,151 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.small.0-1731847662151,5,FailOnTimeoutGroup]
2024-11-17T12:47:42,151 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T12:47:42,151 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-17T12:47:42,151 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-17T12:47:42,151 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-17T12:47:42,151 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:47:42,151 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-17T12:47:42,158 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34729 is added to blk_1073741831_1007 (size=1321)
2024-11-17T12:47:42,158 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36187 is added to blk_1073741831_1007 (size=1321)
2024-11-17T12:47:42,159 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-17T12:47:42,159 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb
2024-11-17T12:47:42,168 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36187 is added to blk_1073741832_1008 (size=32)
2024-11-17T12:47:42,168 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34729 is added to blk_1073741832_1008 (size=32)
2024-11-17T12:47:42,169 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:47:42,171 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T12:47:42,172 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T12:47:42,172 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:47:42,173 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:47:42,173 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T12:47:42,174 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T12:47:42,174 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:47:42,174 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:47:42,174 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T12:47:42,176 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T12:47:42,176 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:47:42,176 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:47:42,176 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T12:47:42,177 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T12:47:42,178 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:47:42,178 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:47:42,178 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T12:47:42,179 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/hbase/meta/1588230740
2024-11-17T12:47:42,179 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/hbase/meta/1588230740
2024-11-17T12:47:42,180 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T12:47:42,180 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T12:47:42,181 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T12:47:42,182 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T12:47:42,184 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T12:47:42,185 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=872743, jitterRate=0.10975012183189392}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T12:47:42,186 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731847662169Initializing all the Stores at 1731847662170 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847662170Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847662170Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847662170Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847662170Cleaning up temporary data from old regions at 1731847662180 (+10 ms)Region opened successfully at 1731847662186 (+6 ms)
2024-11-17T12:47:42,186 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T12:47:42,186 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T12:47:42,186 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T12:47:42,186 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T12:47:42,186 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T12:47:42,187 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T12:47:42,187 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731847662186Disabling compacts and flushes for region at 1731847662186Disabling writes for close at 1731847662186Writing region close event to WAL at 1731847662187 (+1 ms)Closed at 1731847662187
2024-11-17T12:47:42,188 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T12:47:42,189 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-17T12:47:42,189 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-17T12:47:42,190 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T12:47:42,191 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-17T12:47:42,197 INFO  [RS:0;0d751a622496:34727 {}] regionserver.HRegionServer(746): ClusterId : 46cb9b08-f5f9-4bd0-a740-c11862549a67
2024-11-17T12:47:42,197 DEBUG [RS:0;0d751a622496:34727 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-17T12:47:42,233 DEBUG [RS:0;0d751a622496:34727 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-17T12:47:42,233 DEBUG [RS:0;0d751a622496:34727 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-17T12:47:42,241 DEBUG [RS:0;0d751a622496:34727 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-17T12:47:42,242 DEBUG [RS:0;0d751a622496:34727 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24487205, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0d751a622496/172.17.0.2:0
2024-11-17T12:47:42,257 DEBUG [RS:0;0d751a622496:34727 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0d751a622496:34727
2024-11-17T12:47:42,257 INFO  [RS:0;0d751a622496:34727 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-17T12:47:42,257 INFO  [RS:0;0d751a622496:34727 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-17T12:47:42,257 DEBUG [RS:0;0d751a622496:34727 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-17T12:47:42,258 INFO  [RS:0;0d751a622496:34727 {}] regionserver.HRegionServer(2659): reportForDuty to master=0d751a622496,45911,1731847661230 with port=34727, startcode=1731847661375
2024-11-17T12:47:42,258 DEBUG [RS:0;0d751a622496:34727 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-17T12:47:42,260 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46331, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-11-17T12:47:42,260 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45911 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0d751a622496,34727,1731847661375
2024-11-17T12:47:42,260 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45911 {}] master.ServerManager(517): Registering regionserver=0d751a622496,34727,1731847661375
2024-11-17T12:47:42,262 DEBUG [RS:0;0d751a622496:34727 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb
2024-11-17T12:47:42,262 DEBUG [RS:0;0d751a622496:34727 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42599
2024-11-17T12:47:42,262 DEBUG [RS:0;0d751a622496:34727 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-17T12:47:42,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T12:47:42,271 DEBUG [RS:0;0d751a622496:34727 {}] zookeeper.ZKUtil(111): regionserver:34727-0x1014a1fb9610001, quorum=127.0.0.1:60904, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0d751a622496,34727,1731847661375
2024-11-17T12:47:42,271 WARN  [RS:0;0d751a622496:34727 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T12:47:42,271 INFO  [RS:0;0d751a622496:34727 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T12:47:42,271 DEBUG [RS:0;0d751a622496:34727 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375
2024-11-17T12:47:42,271 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0d751a622496,34727,1731847661375]
2024-11-17T12:47:42,274 INFO  [RS:0;0d751a622496:34727 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-17T12:47:42,277 INFO  [RS:0;0d751a622496:34727 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-17T12:47:42,279 INFO  [RS:0;0d751a622496:34727 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-17T12:47:42,279 INFO  [RS:0;0d751a622496:34727 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:47:42,279 INFO  [RS:0;0d751a622496:34727 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-17T12:47:42,280 INFO  [RS:0;0d751a622496:34727 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-17T12:47:42,280 INFO  [RS:0;0d751a622496:34727 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-17T12:47:42,280 DEBUG [RS:0;0d751a622496:34727 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:47:42,281 DEBUG [RS:0;0d751a622496:34727 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:47:42,281 DEBUG [RS:0;0d751a622496:34727 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:47:42,281 DEBUG [RS:0;0d751a622496:34727 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:47:42,281 DEBUG [RS:0;0d751a622496:34727 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:47:42,281 DEBUG [RS:0;0d751a622496:34727 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0, corePoolSize=2, maxPoolSize=2
2024-11-17T12:47:42,281 DEBUG [RS:0;0d751a622496:34727 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:47:42,281 DEBUG [RS:0;0d751a622496:34727 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:47:42,281 DEBUG [RS:0;0d751a622496:34727 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:47:42,281 DEBUG [RS:0;0d751a622496:34727 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:47:42,281 DEBUG [RS:0;0d751a622496:34727 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:47:42,281 DEBUG [RS:0;0d751a622496:34727 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:47:42,281 DEBUG [RS:0;0d751a622496:34727 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0d751a622496:0, corePoolSize=3, maxPoolSize=3
2024-11-17T12:47:42,281 DEBUG [RS:0;0d751a622496:34727 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0, corePoolSize=3, maxPoolSize=3
2024-11-17T12:47:42,282 INFO  [RS:0;0d751a622496:34727 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T12:47:42,282 INFO  [RS:0;0d751a622496:34727 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T12:47:42,282 INFO  [RS:0;0d751a622496:34727 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:47:42,282 INFO  [RS:0;0d751a622496:34727 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-17T12:47:42,282 INFO  [RS:0;0d751a622496:34727 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-17T12:47:42,282 INFO  [RS:0;0d751a622496:34727 {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,34727,1731847661375-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T12:47:42,303 INFO  [RS:0;0d751a622496:34727 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-17T12:47:42,303 INFO  [RS:0;0d751a622496:34727 {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,34727,1731847661375-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:47:42,303 INFO  [RS:0;0d751a622496:34727 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:47:42,303 INFO  [RS:0;0d751a622496:34727 {}] regionserver.Replication(171): 0d751a622496,34727,1731847661375 started
2024-11-17T12:47:42,323 INFO  [RS:0;0d751a622496:34727 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:47:42,323 INFO  [RS:0;0d751a622496:34727 {}] regionserver.HRegionServer(1482): Serving as 0d751a622496,34727,1731847661375, RpcServer on 0d751a622496/172.17.0.2:34727, sessionid=0x1014a1fb9610001
2024-11-17T12:47:42,323 DEBUG [RS:0;0d751a622496:34727 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-17T12:47:42,323 DEBUG [RS:0;0d751a622496:34727 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0d751a622496,34727,1731847661375
2024-11-17T12:47:42,323 DEBUG [RS:0;0d751a622496:34727 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0d751a622496,34727,1731847661375'
2024-11-17T12:47:42,323 DEBUG [RS:0;0d751a622496:34727 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-17T12:47:42,324 DEBUG [RS:0;0d751a622496:34727 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-17T12:47:42,324 DEBUG [RS:0;0d751a622496:34727 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-17T12:47:42,324 DEBUG [RS:0;0d751a622496:34727 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-17T12:47:42,324 DEBUG [RS:0;0d751a622496:34727 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0d751a622496,34727,1731847661375
2024-11-17T12:47:42,324 DEBUG [RS:0;0d751a622496:34727 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0d751a622496,34727,1731847661375'
2024-11-17T12:47:42,324 DEBUG [RS:0;0d751a622496:34727 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-17T12:47:42,325 DEBUG [RS:0;0d751a622496:34727 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-17T12:47:42,325 DEBUG [RS:0;0d751a622496:34727 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-17T12:47:42,325 INFO  [RS:0;0d751a622496:34727 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-17T12:47:42,325 INFO  [RS:0;0d751a622496:34727 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-17T12:47:42,342 WARN  [0d751a622496:45911 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-17T12:47:42,428 INFO  [RS:0;0d751a622496:34727 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C34727%2C1731847661375, suffix=, logDir=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375, archiveDir=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/oldWALs, maxLogs=32
2024-11-17T12:47:42,429 INFO  [RS:0;0d751a622496:34727 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C34727%2C1731847661375.1731847662429
2024-11-17T12:47:42,436 INFO  [RS:0;0d751a622496:34727 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847662429
2024-11-17T12:47:42,437 DEBUG [RS:0;0d751a622496:34727 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35693:35693),(127.0.0.1/127.0.0.1:44187:44187)]
2024-11-17T12:47:42,592 DEBUG [0d751a622496:45911 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-17T12:47:42,593 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0d751a622496,34727,1731847661375
2024-11-17T12:47:42,595 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0d751a622496,34727,1731847661375, state=OPENING
2024-11-17T12:47:42,623 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-17T12:47:42,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:47:42,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34727-0x1014a1fb9610001, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:47:42,632 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T12:47:42,633 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0d751a622496,34727,1731847661375}]
2024-11-17T12:47:42,633 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:47:42,633 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:47:42,773 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:42,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:42,787 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-17T12:47:42,789 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60083, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-17T12:47:42,793 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-17T12:47:42,794 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T12:47:42,796 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C34727%2C1731847661375.meta, suffix=.meta, logDir=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375, archiveDir=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/oldWALs, maxLogs=32
2024-11-17T12:47:42,796 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C34727%2C1731847661375.meta.1731847662796.meta
2024-11-17T12:47:42,805 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.meta.1731847662796.meta
2024-11-17T12:47:42,806 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44187:44187),(127.0.0.1/127.0.0.1:35693:35693)]
2024-11-17T12:47:42,806 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-17T12:47:42,807 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-17T12:47:42,807 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-17T12:47:42,807 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-17T12:47:42,807 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-17T12:47:42,807 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:47:42,807 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-17T12:47:42,807 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-17T12:47:42,808 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T12:47:42,809 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T12:47:42,809 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:47:42,810 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:47:42,810 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T12:47:42,811 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T12:47:42,811 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:47:42,811 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:47:42,811 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T12:47:42,812 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T12:47:42,812 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:47:42,813 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:47:42,813 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T12:47:42,814 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T12:47:42,814 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:47:42,814 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:47:42,814 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T12:47:42,816 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/hbase/meta/1588230740
2024-11-17T12:47:42,817 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/hbase/meta/1588230740
2024-11-17T12:47:42,818 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T12:47:42,818 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T12:47:42,819 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T12:47:42,820 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T12:47:42,821 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=785654, jitterRate=-9.899288415908813E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T12:47:42,822 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-17T12:47:42,822 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731847662807Writing region info on filesystem at 1731847662807Initializing all the Stores at 1731847662808 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847662808Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847662808Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847662808Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847662808Cleaning up temporary data from old regions at 1731847662818 (+10 ms)Running coprocessor post-open hooks at 1731847662822 (+4 ms)Region opened successfully at 1731847662822
2024-11-17T12:47:42,824 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731847662787
2024-11-17T12:47:42,826 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-17T12:47:42,826 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-17T12:47:42,828 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0d751a622496,34727,1731847661375
2024-11-17T12:47:42,829 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0d751a622496,34727,1731847661375, state=OPEN
2024-11-17T12:47:42,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T12:47:42,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34727-0x1014a1fb9610001, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T12:47:42,862 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0d751a622496,34727,1731847661375
2024-11-17T12:47:42,862 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:47:42,862 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:47:42,868 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-17T12:47:42,868 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0d751a622496,34727,1731847661375 in 229 msec
2024-11-17T12:47:42,872 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-17T12:47:42,872 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 679 msec
2024-11-17T12:47:42,873 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T12:47:42,873 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-17T12:47:42,875 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T12:47:42,875 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0d751a622496,34727,1731847661375, seqNum=-1]
2024-11-17T12:47:42,875 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T12:47:42,877 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48221, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T12:47:42,883 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 737 msec
2024-11-17T12:47:42,883 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731847662883, completionTime=-1
2024-11-17T12:47:42,883 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-17T12:47:42,883 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-17T12:47:42,885 INFO  [master/0d751a622496:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-17T12:47:42,885 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731847722885
2024-11-17T12:47:42,885 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731847782885
2024-11-17T12:47:42,885 INFO  [master/0d751a622496:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-17T12:47:42,886 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,45911,1731847661230-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:47:42,886 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,45911,1731847661230-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:47:42,886 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,45911,1731847661230-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:47:42,886 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0d751a622496:45911, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:47:42,886 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-17T12:47:42,886 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-17T12:47:42,888 DEBUG [master/0d751a622496:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-17T12:47:42,891 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.450sec
2024-11-17T12:47:42,891 INFO  [master/0d751a622496:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-17T12:47:42,891 INFO  [master/0d751a622496:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-17T12:47:42,891 INFO  [master/0d751a622496:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-17T12:47:42,891 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-17T12:47:42,891 INFO  [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-17T12:47:42,891 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,45911,1731847661230-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T12:47:42,891 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,45911,1731847661230-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-17T12:47:42,893 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-17T12:47:42,893 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-17T12:47:42,893 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,45911,1731847661230-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:47:42,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e70018, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T12:47:42,898 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0d751a622496,45911,-1 for getting cluster id
2024-11-17T12:47:42,898 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-17T12:47:42,899 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46cb9b08-f5f9-4bd0-a740-c11862549a67'
2024-11-17T12:47:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-17T12:47:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46cb9b08-f5f9-4bd0-a740-c11862549a67"

2024-11-17T12:47:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@767d3f08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T12:47:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0d751a622496,45911,-1]
2024-11-17T12:47:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-17T12:47:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:47:42,902 INFO  [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43752, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-17T12:47:42,903 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dcfabbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T12:47:42,903 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T12:47:42,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0d751a622496,34727,1731847661375, seqNum=-1]
2024-11-17T12:47:42,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T12:47:42,906 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T12:47:42,908 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0d751a622496,45911,1731847661230
2024-11-17T12:47:42,909 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:47:42,911 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-17T12:47:42,911 INFO  [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart
2024-11-17T12:47:42,911 INFO  [Time-limited test {}] wal.TestLogRolling(323): Replication=2
2024-11-17T12:47:42,912 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-17T12:47:42,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is 0d751a622496,45911,1731847661230
2024-11-17T12:47:42,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1a63cfba
2024-11-17T12:47:42,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-17T12:47:42,915 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-17T12:47:42,916 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45911 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-17T12:47:42,916 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45911 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-17T12:47:42,916 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45911 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-17T12:47:42,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45911 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart
2024-11-17T12:47:42,919 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION
2024-11-17T12:47:42,920 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:47:42,920 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45911 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnPipelineRestart"
 procId is: 4
2024-11-17T12:47:42,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45911 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-17T12:47:42,921 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-17T12:47:42,931 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36187 is added to blk_1073741835_1011 (size=395)
2024-11-17T12:47:42,931 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34729 is added to blk_1073741835_1011 (size=395)
2024-11-17T12:47:42,933 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4570c3f095eca1b9f8b53d3b7ef1e29d, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1731847662915.4570c3f095eca1b9f8b53d3b7ef1e29d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb
2024-11-17T12:47:42,939 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36187 is added to blk_1073741836_1012 (size=78)
2024-11-17T12:47:42,940 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34729 is added to blk_1073741836_1012 (size=78)
2024-11-17T12:47:42,940 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1731847662915.4570c3f095eca1b9f8b53d3b7ef1e29d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:47:42,940 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing 4570c3f095eca1b9f8b53d3b7ef1e29d, disabling compactions & flushes
2024-11-17T12:47:42,940 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1731847662915.4570c3f095eca1b9f8b53d3b7ef1e29d.
2024-11-17T12:47:42,940 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731847662915.4570c3f095eca1b9f8b53d3b7ef1e29d.
2024-11-17T12:47:42,940 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731847662915.4570c3f095eca1b9f8b53d3b7ef1e29d. after waiting 0 ms
2024-11-17T12:47:42,940 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1731847662915.4570c3f095eca1b9f8b53d3b7ef1e29d.
2024-11-17T12:47:42,940 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731847662915.4570c3f095eca1b9f8b53d3b7ef1e29d.
2024-11-17T12:47:42,940 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4570c3f095eca1b9f8b53d3b7ef1e29d:
Waiting for close lock at 1731847662940Disabling compacts and flushes for region at 1731847662940Disabling writes for close at 1731847662940Writing region close event to WAL at 1731847662940Closed at 1731847662940
2024-11-17T12:47:42,942 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META
2024-11-17T12:47:42,942 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1731847662915.4570c3f095eca1b9f8b53d3b7ef1e29d.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1731847662942"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731847662942"}]},"ts":"1731847662942"}
2024-11-17T12:47:42,945 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-17T12:47:42,946 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-17T12:47:42,946 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731847662946"}]},"ts":"1731847662946"}
2024-11-17T12:47:42,948 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta
2024-11-17T12:47:42,949 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=4570c3f095eca1b9f8b53d3b7ef1e29d, ASSIGN}]
2024-11-17T12:47:42,950 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=4570c3f095eca1b9f8b53d3b7ef1e29d, ASSIGN
2024-11-17T12:47:42,952 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=4570c3f095eca1b9f8b53d3b7ef1e29d, ASSIGN; state=OFFLINE, location=0d751a622496,34727,1731847661375; forceNewPlan=false, retain=false
2024-11-17T12:47:43,102 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4570c3f095eca1b9f8b53d3b7ef1e29d, regionState=OPENING, regionLocation=0d751a622496,34727,1731847661375
2024-11-17T12:47:43,105 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=4570c3f095eca1b9f8b53d3b7ef1e29d, ASSIGN because future has completed
2024-11-17T12:47:43,106 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4570c3f095eca1b9f8b53d3b7ef1e29d, server=0d751a622496,34727,1731847661375}]
2024-11-17T12:47:43,265 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1731847662915.4570c3f095eca1b9f8b53d3b7ef1e29d.
2024-11-17T12:47:43,265 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 4570c3f095eca1b9f8b53d3b7ef1e29d, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1731847662915.4570c3f095eca1b9f8b53d3b7ef1e29d.', STARTKEY => '', ENDKEY => ''}
2024-11-17T12:47:43,266 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart 4570c3f095eca1b9f8b53d3b7ef1e29d
2024-11-17T12:47:43,266 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1731847662915.4570c3f095eca1b9f8b53d3b7ef1e29d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:47:43,266 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 4570c3f095eca1b9f8b53d3b7ef1e29d
2024-11-17T12:47:43,266 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 4570c3f095eca1b9f8b53d3b7ef1e29d
2024-11-17T12:47:43,268 INFO  [StoreOpener-4570c3f095eca1b9f8b53d3b7ef1e29d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 4570c3f095eca1b9f8b53d3b7ef1e29d 
2024-11-17T12:47:43,270 INFO  [StoreOpener-4570c3f095eca1b9f8b53d3b7ef1e29d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4570c3f095eca1b9f8b53d3b7ef1e29d columnFamilyName info
2024-11-17T12:47:43,270 DEBUG [StoreOpener-4570c3f095eca1b9f8b53d3b7ef1e29d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:47:43,271 INFO  [StoreOpener-4570c3f095eca1b9f8b53d3b7ef1e29d-1 {}] regionserver.HStore(327): Store=4570c3f095eca1b9f8b53d3b7ef1e29d/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:47:43,271 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 4570c3f095eca1b9f8b53d3b7ef1e29d
2024-11-17T12:47:43,272 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/default/TestLogRolling-testLogRollOnPipelineRestart/4570c3f095eca1b9f8b53d3b7ef1e29d
2024-11-17T12:47:43,272 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/default/TestLogRolling-testLogRollOnPipelineRestart/4570c3f095eca1b9f8b53d3b7ef1e29d
2024-11-17T12:47:43,273 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 4570c3f095eca1b9f8b53d3b7ef1e29d
2024-11-17T12:47:43,273 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 4570c3f095eca1b9f8b53d3b7ef1e29d
2024-11-17T12:47:43,275 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 4570c3f095eca1b9f8b53d3b7ef1e29d
2024-11-17T12:47:43,278 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/default/TestLogRolling-testLogRollOnPipelineRestart/4570c3f095eca1b9f8b53d3b7ef1e29d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T12:47:43,278 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 4570c3f095eca1b9f8b53d3b7ef1e29d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=834172, jitterRate=0.06070563197135925}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-17T12:47:43,278 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4570c3f095eca1b9f8b53d3b7ef1e29d
2024-11-17T12:47:43,279 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 4570c3f095eca1b9f8b53d3b7ef1e29d:
Running coprocessor pre-open hook at 1731847663266Writing region info on filesystem at 1731847663266Initializing all the Stores at 1731847663268 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847663268Cleaning up temporary data from old regions at 1731847663273 (+5 ms)Running coprocessor post-open hooks at 1731847663278 (+5 ms)Region opened successfully at 1731847663279 (+1 ms)
2024-11-17T12:47:43,281 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1731847662915.4570c3f095eca1b9f8b53d3b7ef1e29d., pid=6, masterSystemTime=1731847663259
2024-11-17T12:47:43,283 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1731847662915.4570c3f095eca1b9f8b53d3b7ef1e29d.
2024-11-17T12:47:43,283 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1731847662915.4570c3f095eca1b9f8b53d3b7ef1e29d.
2024-11-17T12:47:43,284 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4570c3f095eca1b9f8b53d3b7ef1e29d, regionState=OPEN, openSeqNum=2, regionLocation=0d751a622496,34727,1731847661375
2024-11-17T12:47:43,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4570c3f095eca1b9f8b53d3b7ef1e29d, server=0d751a622496,34727,1731847661375 because future has completed
2024-11-17T12:47:43,291 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-17T12:47:43,291 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4570c3f095eca1b9f8b53d3b7ef1e29d, server=0d751a622496,34727,1731847661375 in 182 msec
2024-11-17T12:47:43,294 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-17T12:47:43,294 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=4570c3f095eca1b9f8b53d3b7ef1e29d, ASSIGN in 343 msec
2024-11-17T12:47:43,295 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-17T12:47:43,296 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731847663295"}]},"ts":"1731847663295"}
2024-11-17T12:47:43,298 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta
2024-11-17T12:47:43,299 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION
2024-11-17T12:47:43,301 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 383 msec
2024-11-17T12:47:43,774 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:43,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:44,775 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:44,780 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:45,776 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:45,781 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:45,983 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-17T12:47:46,008 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:47:46,009 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:47:46,009 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:47:46,009 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:47:46,009 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:47:46,010 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:47:46,013 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:47:46,013 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:47:46,013 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:47:46,016 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:47:46,776 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:46,781 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:47,777 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:47,782 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:48,275 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-17T12:47:48,442 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart'
2024-11-17T12:47:48,778 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:48,782 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:49,778 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:49,783 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:50,479 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-17T12:47:50,480 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-17T12:47:50,480 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-17T12:47:50,481 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer
2024-11-17T12:47:50,481 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T12:47:50,481 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-17T12:47:50,481 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-17T12:47:50,481 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-17T12:47:50,779 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:50,783 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:51,780 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:51,784 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:52,781 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:52,785 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:52,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45911 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-17T12:47:52,970 INFO  [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed
2024-11-17T12:47:52,970 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100
2024-11-17T12:47:52,974 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart
2024-11-17T12:47:52,975 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1731847662915.4570c3f095eca1b9f8b53d3b7ef1e29d.
2024-11-17T12:47:52,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1731847662915.4570c3f095eca1b9f8b53d3b7ef1e29d., hostname=0d751a622496,34727,1731847661375, seqNum=2]
2024-11-17T12:47:53,781 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:53,785 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:54,782 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:54,786 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:54,984 INFO  [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847662429
2024-11-17T12:47:54,985 WARN  [ResponseProcessor for block BP-1897123262-172.17.0.2-1731847659391:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1897123262-172.17.0.2-1731847659391:blk_1073741830_1006
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:54,985 WARN  [ResponseProcessor for block BP-1897123262-172.17.0.2-1731847659391:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1897123262-172.17.0.2-1731847659391:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-1897123262-172.17.0.2-1731847659391:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:34729,DS-c108bbc1-7b60-49b7-be4e-ffef61fc3522,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:54,987 WARN  [DataStreamer for file /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.meta.1731847662796.meta block BP-1897123262-172.17.0.2-1731847659391:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1897123262-172.17.0.2-1731847659391:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36187,DS-40ff578d-b4b3-4910-8233-f15f8ec9c022,DISK], DatanodeInfoWithStorage[127.0.0.1:34729,DS-c108bbc1-7b60-49b7-be4e-ffef61fc3522,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34729,DS-c108bbc1-7b60-49b7-be4e-ffef61fc3522,DISK]) is bad.
2024-11-17T12:47:54,987 WARN  [ResponseProcessor for block BP-1897123262-172.17.0.2-1731847659391:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1897123262-172.17.0.2-1731847659391:blk_1073741833_1009
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:54,987 WARN  [DataStreamer for file /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847662429 block BP-1897123262-172.17.0.2-1731847659391:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1897123262-172.17.0.2-1731847659391:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34729,DS-c108bbc1-7b60-49b7-be4e-ffef61fc3522,DISK], DatanodeInfoWithStorage[127.0.0.1:36187,DS-40ff578d-b4b3-4910-8233-f15f8ec9c022,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34729,DS-c108bbc1-7b60-49b7-be4e-ffef61fc3522,DISK]) is bad.
2024-11-17T12:47:54,987 WARN  [DataStreamer for file /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/WALs/0d751a622496,45911,1731847661230/0d751a622496%2C45911%2C1731847661230.1731847661914 block BP-1897123262-172.17.0.2-1731847659391:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1897123262-172.17.0.2-1731847659391:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34729,DS-c108bbc1-7b60-49b7-be4e-ffef61fc3522,DISK], DatanodeInfoWithStorage[127.0.0.1:36187,DS-40ff578d-b4b3-4910-8233-f15f8ec9c022,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34729,DS-c108bbc1-7b60-49b7-be4e-ffef61fc3522,DISK]) is bad.
2024-11-17T12:47:54,988 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1484360581_22 at /127.0.0.1:48424 [Receiving block BP-1897123262-172.17.0.2-1731847659391:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:34729:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48424 dst: /127.0.0.1:34729
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:54,988 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1600132096_22 at /127.0.0.1:48456 [Receiving block BP-1897123262-172.17.0.2-1731847659391:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:34729:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48456 dst: /127.0.0.1:34729
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:54,988 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1600132096_22 at /127.0.0.1:49738 [Receiving block BP-1897123262-172.17.0.2-1731847659391:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:36187:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49738 dst: /127.0.0.1:36187
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:54,988 WARN  [PacketResponder: BP-1897123262-172.17.0.2-1731847659391:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:34729] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:54,988 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1484360581_22 at /127.0.0.1:49710 [Receiving block BP-1897123262-172.17.0.2-1731847659391:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:36187:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49710 dst: /127.0.0.1:36187
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:54,988 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1600132096_22 at /127.0.0.1:49750 [Receiving block BP-1897123262-172.17.0.2-1731847659391:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:36187:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49750 dst: /127.0.0.1:36187
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:54,988 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1600132096_22 at /127.0.0.1:48470 [Receiving block BP-1897123262-172.17.0.2-1731847659391:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:34729:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48470 dst: /127.0.0.1:34729
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:55,029 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1a8d3987{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:47:55,029 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3e58ccd6{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:47:55,029 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:47:55,030 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@f09ac85{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:47:55,030 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70644e5e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/hadoop.log.dir/,STOPPED}
2024-11-17T12:47:55,032 WARN  [BP-1897123262-172.17.0.2-1731847659391 heartbeating to localhost/127.0.0.1:42599 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T12:47:55,032 WARN  [BP-1897123262-172.17.0.2-1731847659391 heartbeating to localhost/127.0.0.1:42599 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1897123262-172.17.0.2-1731847659391 (Datanode Uuid 4d688259-a8bb-437b-95bd-a86e6fe826d7) service to localhost/127.0.0.1:42599
2024-11-17T12:47:55,033 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/cluster_b93fc69f-3ebf-86a6-e261-c77f3932f38f/data/data4/current/BP-1897123262-172.17.0.2-1731847659391 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:47:55,034 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T12:47:55,034 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T12:47:55,034 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T12:47:55,034 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/cluster_b93fc69f-3ebf-86a6-e261-c77f3932f38f/data/data3/current/BP-1897123262-172.17.0.2-1731847659391 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:47:55,047 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:47:55,051 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:47:55,053 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:47:55,053 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:47:55,053 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T12:47:55,053 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@66943013{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:47:55,054 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@362ea065{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:47:55,186 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c02d1f3{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/java.io.tmpdir/jetty-localhost-38715-hadoop-hdfs-3_4_1-tests_jar-_-any-13738167486685211158/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:47:55,186 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@760c54cf{HTTP/1.1, (http/1.1)}{localhost:38715}
2024-11-17T12:47:55,186 INFO  [Time-limited test {}] server.Server(415): Started @170428ms
2024-11-17T12:47:55,188 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T12:47:55,237 WARN  [ResponseProcessor for block BP-1897123262-172.17.0.2-1731847659391:blk_1073741834_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1897123262-172.17.0.2-1731847659391:blk_1073741834_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:55,238 WARN  [ResponseProcessor for block BP-1897123262-172.17.0.2-1731847659391:blk_1073741833_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1897123262-172.17.0.2-1731847659391:blk_1073741833_1014
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:55,238 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1600132096_22 at /127.0.0.1:40536 [Receiving block BP-1897123262-172.17.0.2-1731847659391:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:36187:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40536 dst: /127.0.0.1:36187
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:55,238 WARN  [ResponseProcessor for block BP-1897123262-172.17.0.2-1731847659391:blk_1073741830_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1897123262-172.17.0.2-1731847659391:blk_1073741830_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:55,238 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1600132096_22 at /127.0.0.1:40522 [Receiving block BP-1897123262-172.17.0.2-1731847659391:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:36187:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40522 dst: /127.0.0.1:36187
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:55,238 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1484360581_22 at /127.0.0.1:40548 [Receiving block BP-1897123262-172.17.0.2-1731847659391:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:36187:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40548 dst: /127.0.0.1:36187
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:47:55,250 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1b91ed3f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:47:55,251 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@16ec9c96{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:47:55,251 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:47:55,251 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@78745d04{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:47:55,251 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@78725bd6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/hadoop.log.dir/,STOPPED}
2024-11-17T12:47:55,252 WARN  [BP-1897123262-172.17.0.2-1731847659391 heartbeating to localhost/127.0.0.1:42599 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T12:47:55,252 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T12:47:55,253 WARN  [BP-1897123262-172.17.0.2-1731847659391 heartbeating to localhost/127.0.0.1:42599 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1897123262-172.17.0.2-1731847659391 (Datanode Uuid 75852f8c-5eae-4c94-85d9-124ca4775ed8) service to localhost/127.0.0.1:42599
2024-11-17T12:47:55,253 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T12:47:55,253 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/cluster_b93fc69f-3ebf-86a6-e261-c77f3932f38f/data/data1/current/BP-1897123262-172.17.0.2-1731847659391 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:47:55,253 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/cluster_b93fc69f-3ebf-86a6-e261-c77f3932f38f/data/data2/current/BP-1897123262-172.17.0.2-1731847659391 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:47:55,253 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T12:47:55,272 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:47:55,280 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:47:55,291 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:47:55,291 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:47:55,291 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T12:47:55,292 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6e1f4e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:47:55,292 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@262fed34{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:47:55,416 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6573e60c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/java.io.tmpdir/jetty-localhost-46763-hadoop-hdfs-3_4_1-tests_jar-_-any-3653208753125164858/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:47:55,417 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3e7ae1c2{HTTP/1.1, (http/1.1)}{localhost:46763}
2024-11-17T12:47:55,417 INFO  [Time-limited test {}] server.Server(415): Started @170658ms
2024-11-17T12:47:55,418 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T12:47:55,576 WARN  [Thread-1336 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T12:47:55,580 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdbe4ee218cfd860 with lease ID 0xd0c3c5f11ba5aa5c: from storage DS-c108bbc1-7b60-49b7-be4e-ffef61fc3522 node DatanodeRegistration(127.0.0.1:33281, datanodeUuid=4d688259-a8bb-437b-95bd-a86e6fe826d7, infoPort=35933, infoSecurePort=0, ipcPort=43573, storageInfo=lv=-57;cid=testClusterID;nsid=1105269691;c=1731847659391), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-17T12:47:55,581 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdbe4ee218cfd860 with lease ID 0xd0c3c5f11ba5aa5c: from storage DS-b03802a7-7ed7-4def-ae03-0ad8cd494bc0 node DatanodeRegistration(127.0.0.1:33281, datanodeUuid=4d688259-a8bb-437b-95bd-a86e6fe826d7, infoPort=35933, infoSecurePort=0, ipcPort=43573, storageInfo=lv=-57;cid=testClusterID;nsid=1105269691;c=1731847659391), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:47:55,783 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:55,787 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:55,799 WARN  [Thread-1357 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T12:47:55,802 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc07565fd24b547d0 with lease ID 0xd0c3c5f11ba5aa5d: from storage DS-40ff578d-b4b3-4910-8233-f15f8ec9c022 node DatanodeRegistration(127.0.0.1:41685, datanodeUuid=75852f8c-5eae-4c94-85d9-124ca4775ed8, infoPort=33545, infoSecurePort=0, ipcPort=45721, storageInfo=lv=-57;cid=testClusterID;nsid=1105269691;c=1731847659391), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-17T12:47:55,803 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc07565fd24b547d0 with lease ID 0xd0c3c5f11ba5aa5d: from storage DS-dcb3d0c4-a239-4bff-83f0-5aec39a96b0a node DatanodeRegistration(127.0.0.1:41685, datanodeUuid=75852f8c-5eae-4c94-85d9-124ca4775ed8, infoPort=33545, infoSecurePort=0, ipcPort=45721, storageInfo=lv=-57;cid=testClusterID;nsid=1105269691;c=1731847659391), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:47:56,445 INFO  [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted
2024-11-17T12:47:56,448 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002
2024-11-17T12:47:56,449 ERROR [FSHLog-0-hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb-prefix:0d751a622496,34727,1731847661375 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36187,DS-40ff578d-b4b3-4910-8233-f15f8ec9c022,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:56,449 WARN  [FSHLog-0-hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb-prefix:0d751a622496,34727,1731847661375 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36187,DS-40ff578d-b4b3-4910-8233-f15f8ec9c022,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:56,450 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0d751a622496%2C34727%2C1731847661375:(num 1731847662429) roll requested
2024-11-17T12:47:56,450 INFO  [regionserver/0d751a622496:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C34727%2C1731847661375.1731847676450
2024-11-17T12:47:56,492 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847662429 newFile=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847676450
2024-11-17T12:47:56,492 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:56,492 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:56,492 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:56,492 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:56,492 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:47:56,492 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847662429 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847676450
2024-11-17T12:47:56,493 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36187,DS-40ff578d-b4b3-4910-8233-f15f8ec9c022,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:56,493 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36187,DS-40ff578d-b4b3-4910-8233-f15f8ec9c022,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:47:56,493 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847662429
2024-11-17T12:47:56,494 WARN  [IPC Server handler 3 on default port 42599 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847662429 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1014
2024-11-17T12:47:56,494 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847662429 after 1ms
2024-11-17T12:47:56,494 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35933:35933),(127.0.0.1/127.0.0.1:33545:33545)]
2024-11-17T12:47:56,494 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847662429 is not closed yet, will try archiving it next time
2024-11-17T12:47:56,783 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:56,790 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:57,784 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:57,791 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:58,498 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003
2024-11-17T12:47:58,581 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741833_1014: GenerationStamp not matched, existing replica is blk_1073741833_1009
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-17T12:47:58,785 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:58,792 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:59,785 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:47:59,792 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:00,494 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847662429 after 4001ms
2024-11-17T12:48:00,501 WARN  [ResponseProcessor for block BP-1897123262-172.17.0.2-1731847659391:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1897123262-172.17.0.2-1731847659391:blk_1073741837_1016
java.io.IOException: Bad response ERROR for BP-1897123262-172.17.0.2-1731847659391:blk_1073741837_1016 from datanode DatanodeInfoWithStorage[127.0.0.1:41685,DS-40ff578d-b4b3-4910-8233-f15f8ec9c022,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:48:00,502 WARN  [DataStreamer for file /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847676450 block BP-1897123262-172.17.0.2-1731847659391:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1897123262-172.17.0.2-1731847659391:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33281,DS-c108bbc1-7b60-49b7-be4e-ffef61fc3522,DISK], DatanodeInfoWithStorage[127.0.0.1:41685,DS-40ff578d-b4b3-4910-8233-f15f8ec9c022,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41685,DS-40ff578d-b4b3-4910-8233-f15f8ec9c022,DISK]) is bad.
2024-11-17T12:48:00,502 WARN  [PacketResponder: BP-1897123262-172.17.0.2-1731847659391:blk_1073741837_1016, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:41685] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:48:00,502 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1600132096_22 at /127.0.0.1:55742 [Receiving block BP-1897123262-172.17.0.2-1731847659391:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:33281:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:55742 dst: /127.0.0.1:33281
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:48:00,502 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1600132096_22 at /127.0.0.1:58062 [Receiving block BP-1897123262-172.17.0.2-1731847659391:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:41685:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58062 dst: /127.0.0.1:41685
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:48:00,551 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6573e60c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:48:00,551 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3e7ae1c2{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:48:00,551 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:48:00,552 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@262fed34{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:48:00,552 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6e1f4e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/hadoop.log.dir/,STOPPED}
2024-11-17T12:48:00,555 WARN  [BP-1897123262-172.17.0.2-1731847659391 heartbeating to localhost/127.0.0.1:42599 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T12:48:00,555 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T12:48:00,555 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T12:48:00,555 WARN  [BP-1897123262-172.17.0.2-1731847659391 heartbeating to localhost/127.0.0.1:42599 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1897123262-172.17.0.2-1731847659391 (Datanode Uuid 75852f8c-5eae-4c94-85d9-124ca4775ed8) service to localhost/127.0.0.1:42599
2024-11-17T12:48:00,555 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/cluster_b93fc69f-3ebf-86a6-e261-c77f3932f38f/data/data1/current/BP-1897123262-172.17.0.2-1731847659391 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:48:00,556 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/cluster_b93fc69f-3ebf-86a6-e261-c77f3932f38f/data/data2/current/BP-1897123262-172.17.0.2-1731847659391 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:48:00,556 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T12:48:00,565 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:48:00,570 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:48:00,570 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:48:00,571 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:48:00,571 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T12:48:00,571 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6ba84765{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:48:00,572 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5d459d02{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:48:00,683 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2da9d855{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/java.io.tmpdir/jetty-localhost-41503-hadoop-hdfs-3_4_1-tests_jar-_-any-787400351977414031/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:48:00,683 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4e60361d{HTTP/1.1, (http/1.1)}{localhost:41503}
2024-11-17T12:48:00,684 INFO  [Time-limited test {}] server.Server(415): Started @175925ms
2024-11-17T12:48:00,685 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T12:48:00,727 WARN  [ResponseProcessor for block BP-1897123262-172.17.0.2-1731847659391:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1897123262-172.17.0.2-1731847659391:blk_1073741837_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:48:00,728 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1600132096_22 at /127.0.0.1:55758 [Receiving block BP-1897123262-172.17.0.2-1731847659391:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:33281:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:55758 dst: /127.0.0.1:33281
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:48:00,729 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c02d1f3{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:48:00,729 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@760c54cf{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:48:00,729 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:48:00,730 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@362ea065{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:48:00,730 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@66943013{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/hadoop.log.dir/,STOPPED}
2024-11-17T12:48:00,731 WARN  [BP-1897123262-172.17.0.2-1731847659391 heartbeating to localhost/127.0.0.1:42599 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T12:48:00,731 WARN  [BP-1897123262-172.17.0.2-1731847659391 heartbeating to localhost/127.0.0.1:42599 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1897123262-172.17.0.2-1731847659391 (Datanode Uuid 4d688259-a8bb-437b-95bd-a86e6fe826d7) service to localhost/127.0.0.1:42599
2024-11-17T12:48:00,731 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T12:48:00,731 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T12:48:00,731 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/cluster_b93fc69f-3ebf-86a6-e261-c77f3932f38f/data/data3/current/BP-1897123262-172.17.0.2-1731847659391 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:48:00,732 WARN  [Time-limited test {}] datanode.FileIoProvider(249): Failed to delete file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/cluster_b93fc69f-3ebf-86a6-e261-c77f3932f38f/data/data4/current/BP-1897123262-172.17.0.2-1731847659391/current/dfsUsed
2024-11-17T12:48:00,732 WARN  [Time-limited test {}] impl.BlockPoolSlice(395): Failed to delete old dfsUsed file in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/cluster_b93fc69f-3ebf-86a6-e261-c77f3932f38f/data/data4/current/BP-1897123262-172.17.0.2-1731847659391/current
2024-11-17T12:48:00,732 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/cluster_b93fc69f-3ebf-86a6-e261-c77f3932f38f/data/data4/current/BP-1897123262-172.17.0.2-1731847659391 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:48:00,732 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T12:48:00,758 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:48:00,762 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:48:00,767 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:48:00,767 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:48:00,767 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T12:48:00,767 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@157f7042{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:48:00,768 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@78489204{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:48:00,786 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:00,793 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:00,876 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@40c9d28{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/java.io.tmpdir/jetty-localhost-37119-hadoop-hdfs-3_4_1-tests_jar-_-any-13592776091280652520/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:48:00,877 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@53bcf6e3{HTTP/1.1, (http/1.1)}{localhost:37119}
2024-11-17T12:48:00,877 INFO  [Time-limited test {}] server.Server(415): Started @176118ms
2024-11-17T12:48:00,879 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T12:48:01,108 WARN  [Thread-1410 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T12:48:01,111 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe4d0144773109349 with lease ID 0xd0c3c5f11ba5aa5e: from storage DS-40ff578d-b4b3-4910-8233-f15f8ec9c022 node DatanodeRegistration(127.0.0.1:42393, datanodeUuid=75852f8c-5eae-4c94-85d9-124ca4775ed8, infoPort=46123, infoSecurePort=0, ipcPort=33219, storageInfo=lv=-57;cid=testClusterID;nsid=1105269691;c=1731847659391), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:48:01,111 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe4d0144773109349 with lease ID 0xd0c3c5f11ba5aa5e: from storage DS-dcb3d0c4-a239-4bff-83f0-5aec39a96b0a node DatanodeRegistration(127.0.0.1:42393, datanodeUuid=75852f8c-5eae-4c94-85d9-124ca4775ed8, infoPort=46123, infoSecurePort=0, ipcPort=33219, storageInfo=lv=-57;cid=testClusterID;nsid=1105269691;c=1731847659391), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:48:01,246 WARN  [Thread-1430 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T12:48:01,248 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc742a3aed5d37ad7 with lease ID 0xd0c3c5f11ba5aa5f: from storage DS-c108bbc1-7b60-49b7-be4e-ffef61fc3522 node DatanodeRegistration(127.0.0.1:41777, datanodeUuid=4d688259-a8bb-437b-95bd-a86e6fe826d7, infoPort=41483, infoSecurePort=0, ipcPort=36891, storageInfo=lv=-57;cid=testClusterID;nsid=1105269691;c=1731847659391), blocks: 7, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-17T12:48:01,249 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc742a3aed5d37ad7 with lease ID 0xd0c3c5f11ba5aa5f: from storage DS-b03802a7-7ed7-4def-ae03-0ad8cd494bc0 node DatanodeRegistration(127.0.0.1:41777, datanodeUuid=4d688259-a8bb-437b-95bd-a86e6fe826d7, infoPort=41483, infoSecurePort=0, ipcPort=36891, storageInfo=lv=-57;cid=testClusterID;nsid=1105269691;c=1731847659391), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:48:01,787 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:01,794 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:01,901 INFO  [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted
2024-11-17T12:48:01,903 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004
2024-11-17T12:48:01,904 ERROR [FSHLog-0-hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb-prefix:0d751a622496,34727,1731847661375 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33281,DS-c108bbc1-7b60-49b7-be4e-ffef61fc3522,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:48:01,905 WARN  [FSHLog-0-hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb-prefix:0d751a622496,34727,1731847661375 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33281,DS-c108bbc1-7b60-49b7-be4e-ffef61fc3522,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:48:01,905 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0d751a622496%2C34727%2C1731847661375:(num 1731847676450) roll requested
2024-11-17T12:48:01,905 INFO  [regionserver/0d751a622496:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C34727%2C1731847661375.1731847681905
2024-11-17T12:48:01,910 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847676450 newFile=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847681905
2024-11-17T12:48:01,910 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:01,910 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:01,910 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:01,910 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:01,910 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:01,911 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847676450 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847681905
2024-11-17T12:48:01,911 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33281,DS-c108bbc1-7b60-49b7-be4e-ffef61fc3522,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:48:01,911 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33281,DS-c108bbc1-7b60-49b7-be4e-ffef61fc3522,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:48:01,911 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847676450
2024-11-17T12:48:01,911 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41483:41483),(127.0.0.1/127.0.0.1:46123:46123)]
2024-11-17T12:48:01,911 WARN  [IPC Server handler 1 on default port 42599 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847676450 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018
2024-11-17T12:48:01,911 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847676450 is not closed yet, will try archiving it next time
2024-11-17T12:48:01,912 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847676450 after 0ms
2024-11-17T12:48:02,788 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:02,794 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:03,788 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:03,795 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:03,913 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C34727%2C1731847661375.1731847683912
2024-11-17T12:48:03,919 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847681905 newFile=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847683912
2024-11-17T12:48:03,920 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:03,920 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:03,920 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:03,920 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:03,920 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:03,920 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847681905 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847683912
2024-11-17T12:48:03,922 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741838_1019 (size=1264)
2024-11-17T12:48:03,923 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41777 is added to blk_1073741838_1019 (size=1264)
2024-11-17T12:48:03,923 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46123:46123),(127.0.0.1/127.0.0.1:41483:41483)]
2024-11-17T12:48:03,923 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847676450 is not closed yet, will try archiving it next time
2024-11-17T12:48:03,924 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847676450 is not closed yet, will try archiving it next time
2024-11-17T12:48:03,924 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847662429
2024-11-17T12:48:03,924 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847662429
2024-11-17T12:48:03,924 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847662429 after 0ms
2024-11-17T12:48:03,925 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847662429
2024-11-17T12:48:03,935 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1731847663279/Put/vlen=218/seqid=0]
2024-11-17T12:48:03,935 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1731847672982/Put/vlen=1045/seqid=0]
2024-11-17T12:48:03,936 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847662429
2024-11-17T12:48:03,936 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847676450
2024-11-17T12:48:03,936 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847676450
2024-11-17T12:48:03,936 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847676450 after 0ms
2024-11-17T12:48:03,936 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847676450
2024-11-17T12:48:03,940 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1731847676449/Put/vlen=1045/seqid=0]
2024-11-17T12:48:03,940 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1731847678499/Put/vlen=1045/seqid=0]
2024-11-17T12:48:03,940 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847676450
2024-11-17T12:48:03,941 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847681905
2024-11-17T12:48:03,941 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847681905
2024-11-17T12:48:03,941 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847681905 after 0ms
2024-11-17T12:48:03,941 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847681905
2024-11-17T12:48:03,945 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1731847681904/Put/vlen=1045/seqid=0]
2024-11-17T12:48:03,945 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847683912
2024-11-17T12:48:03,945 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847683912
2024-11-17T12:48:03,946 WARN  [IPC Server handler 0 on default port 42599 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847683912 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021
2024-11-17T12:48:03,946 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847683912 after 1ms
2024-11-17T12:48:04,111 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-17T12:48:04,253 WARN  [ResponseProcessor for block BP-1897123262-172.17.0.2-1731847659391:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1897123262-172.17.0.2-1731847659391:blk_1073741839_1021
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:48:04,253 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1484360581_22 at /127.0.0.1:55788 [Receiving block BP-1897123262-172.17.0.2-1731847659391:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:42393:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:55788 dst: /127.0.0.1:42393
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:42393 remote=/127.0.0.1:55788]. Total timeout mills is 60000, 59666 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:48:04,254 WARN  [DataStreamer for file /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847683912 block BP-1897123262-172.17.0.2-1731847659391:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1897123262-172.17.0.2-1731847659391:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42393,DS-40ff578d-b4b3-4910-8233-f15f8ec9c022,DISK], DatanodeInfoWithStorage[127.0.0.1:41777,DS-c108bbc1-7b60-49b7-be4e-ffef61fc3522,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42393,DS-40ff578d-b4b3-4910-8233-f15f8ec9c022,DISK]) is bad.
2024-11-17T12:48:04,254 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1484360581_22 at /127.0.0.1:48802 [Receiving block BP-1897123262-172.17.0.2-1731847659391:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:41777:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48802 dst: /127.0.0.1:41777
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:48:04,255 WARN  [DataStreamer for file /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847683912 block BP-1897123262-172.17.0.2-1731847659391:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1897123262-172.17.0.2-1731847659391:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:48:04,260 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741839_1022 (size=85)
2024-11-17T12:48:04,789 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:04,796 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:05,790 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:05,796 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:05,913 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847676450 after 4002ms
2024-11-17T12:48:06,791 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:06,797 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:07,792 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:07,797 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:07,947 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847683912 after 4002ms
2024-11-17T12:48:07,947 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847683912
2024-11-17T12:48:07,951 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847683912
2024-11-17T12:48:07,951 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 4570c3f095eca1b9f8b53d3b7ef1e29d 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-11-17T12:48:07,952 ERROR [FSHLog-0-hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb-prefix:0d751a622496,34727,1731847661375 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1897123262-172.17.0.2-1731847659391:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:48:07,952 WARN  [FSHLog-0-hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb-prefix:0d751a622496,34727,1731847661375 {}] wal.AbstractFSWAL(2174): append entry failed
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1897123262-172.17.0.2-1731847659391:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:48:07,952 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0d751a622496%2C34727%2C1731847661375:(num 1731847683912) roll requested
2024-11-17T12:48:07,953 INFO  [regionserver/0d751a622496:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C34727%2C1731847661375.1731847687953
2024-11-17T12:48:07,961 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847683912 newFile=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847687953
2024-11-17T12:48:07,962 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:07,962 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:07,962 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:07,962 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:07,962 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:07,962 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847683912 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847687953
2024-11-17T12:48:07,962 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1897123262-172.17.0.2-1731847659391:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:48:07,963 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46123:46123),(127.0.0.1/127.0.0.1:41483:41483)]
2024-11-17T12:48:07,963 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847683912 is not closed yet, will try archiving it next time
2024-11-17T12:48:07,963 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1897123262-172.17.0.2-1731847659391:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:48:07,963 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847683912
2024-11-17T12:48:07,964 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847683912 after 1ms
2024-11-17T12:48:07,964 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.1731847683912 to hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/oldWALs/0d751a622496%2C34727%2C1731847661375.1731847683912
2024-11-17T12:48:07,981 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/default/TestLogRolling-testLogRollOnPipelineRestart/4570c3f095eca1b9f8b53d3b7ef1e29d/.tmp/info/61f37ce00678456fa023e4fca088c7cd is 1080, key is row1002/info:/1731847672982/Put/seqid=0
2024-11-17T12:48:07,988 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741841_1024 (size=9270)
2024-11-17T12:48:07,989 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41777 is added to blk_1073741841_1024 (size=9270)
2024-11-17T12:48:07,989 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/default/TestLogRolling-testLogRollOnPipelineRestart/4570c3f095eca1b9f8b53d3b7ef1e29d/.tmp/info/61f37ce00678456fa023e4fca088c7cd
2024-11-17T12:48:07,996 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/default/TestLogRolling-testLogRollOnPipelineRestart/4570c3f095eca1b9f8b53d3b7ef1e29d/.tmp/info/61f37ce00678456fa023e4fca088c7cd as hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/default/TestLogRolling-testLogRollOnPipelineRestart/4570c3f095eca1b9f8b53d3b7ef1e29d/info/61f37ce00678456fa023e4fca088c7cd
2024-11-17T12:48:08,003 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/default/TestLogRolling-testLogRollOnPipelineRestart/4570c3f095eca1b9f8b53d3b7ef1e29d/info/61f37ce00678456fa023e4fca088c7cd, entries=4, sequenceid=8, filesize=9.1 K
2024-11-17T12:48:08,005 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 4570c3f095eca1b9f8b53d3b7ef1e29d in 53ms, sequenceid=8, compaction requested=false
2024-11-17T12:48:08,005 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4570c3f095eca1b9f8b53d3b7ef1e29d:

2024-11-17T12:48:08,005 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB
2024-11-17T12:48:08,005 ERROR [FSHLog-0-hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb-prefix:0d751a622496,34727,1731847661375.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36187,DS-40ff578d-b4b3-4910-8233-f15f8ec9c022,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:48:08,005 WARN  [FSHLog-0-hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb-prefix:0d751a622496,34727,1731847661375.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36187,DS-40ff578d-b4b3-4910-8233-f15f8ec9c022,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:48:08,006 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 0d751a622496%2C34727%2C1731847661375.meta:.meta(num 1731847662796) roll requested
2024-11-17T12:48:08,006 INFO  [regionserver/0d751a622496:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C34727%2C1731847661375.meta.1731847688006.meta
2024-11-17T12:48:08,011 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:08,012 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:08,012 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:08,012 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:08,012 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:08,012 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.meta.1731847662796.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.meta.1731847688006.meta
2024-11-17T12:48:08,012 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36187,DS-40ff578d-b4b3-4910-8233-f15f8ec9c022,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:48:08,012 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36187,DS-40ff578d-b4b3-4910-8233-f15f8ec9c022,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:48:08,013 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.meta.1731847662796.meta
2024-11-17T12:48:08,013 WARN  [IPC Server handler 1 on default port 42599 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.meta.1731847662796.meta has not been closed. Lease recovery is in progress. RecoveryId = 1026 for block blk_1073741834_1013
2024-11-17T12:48:08,013 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.meta.1731847662796.meta after 0ms
2024-11-17T12:48:08,019 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41483:41483),(127.0.0.1/127.0.0.1:46123:46123)]
2024-11-17T12:48:08,019 DEBUG [regionserver/0d751a622496:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.meta.1731847662796.meta is not closed yet, will try archiving it next time
2024-11-17T12:48:08,041 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/hbase/meta/1588230740/.tmp/info/915cd1c063c44e3db27e1b31f76ee1e2 is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1731847662915.4570c3f095eca1b9f8b53d3b7ef1e29d./info:regioninfo/1731847663284/Put/seqid=0
2024-11-17T12:48:08,047 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741843_1027 (size=7125)
2024-11-17T12:48:08,047 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41777 is added to blk_1073741843_1027 (size=7125)
2024-11-17T12:48:08,048 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/hbase/meta/1588230740/.tmp/info/915cd1c063c44e3db27e1b31f76ee1e2
2024-11-17T12:48:08,071 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/hbase/meta/1588230740/.tmp/ns/771de9c6dff7407284e2836c9e311be5 is 43, key is default/ns:d/1731847662877/Put/seqid=0
2024-11-17T12:48:08,076 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741844_1028 (size=5153)
2024-11-17T12:48:08,076 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41777 is added to blk_1073741844_1028 (size=5153)
2024-11-17T12:48:08,077 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/hbase/meta/1588230740/.tmp/ns/771de9c6dff7407284e2836c9e311be5
2024-11-17T12:48:08,100 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/hbase/meta/1588230740/.tmp/table/de3b620e38d244e49f8629cd351b5f2e is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1731847663295/Put/seqid=0
2024-11-17T12:48:08,105 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741845_1029 (size=5438)
2024-11-17T12:48:08,105 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41777 is added to blk_1073741845_1029 (size=5438)
2024-11-17T12:48:08,106 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/hbase/meta/1588230740/.tmp/table/de3b620e38d244e49f8629cd351b5f2e
2024-11-17T12:48:08,112 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/hbase/meta/1588230740/.tmp/info/915cd1c063c44e3db27e1b31f76ee1e2 as hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/hbase/meta/1588230740/info/915cd1c063c44e3db27e1b31f76ee1e2
2024-11-17T12:48:08,119 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/hbase/meta/1588230740/info/915cd1c063c44e3db27e1b31f76ee1e2, entries=10, sequenceid=11, filesize=7.0 K
2024-11-17T12:48:08,120 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/hbase/meta/1588230740/.tmp/ns/771de9c6dff7407284e2836c9e311be5 as hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/hbase/meta/1588230740/ns/771de9c6dff7407284e2836c9e311be5
2024-11-17T12:48:08,126 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/hbase/meta/1588230740/ns/771de9c6dff7407284e2836c9e311be5, entries=2, sequenceid=11, filesize=5.0 K
2024-11-17T12:48:08,127 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/hbase/meta/1588230740/.tmp/table/de3b620e38d244e49f8629cd351b5f2e as hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/hbase/meta/1588230740/table/de3b620e38d244e49f8629cd351b5f2e
2024-11-17T12:48:08,133 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/hbase/meta/1588230740/table/de3b620e38d244e49f8629cd351b5f2e, entries=2, sequenceid=11, filesize=5.3 K
2024-11-17T12:48:08,134 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 129ms, sequenceid=11, compaction requested=false
2024-11-17T12:48:08,134 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-17T12:48:08,140 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-17T12:48:08,140 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T12:48:08,140 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T12:48:08,140 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:48:08,140 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:48:08,140 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-17T12:48:08,140 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-17T12:48:08,140 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1250426912, stopped=false
2024-11-17T12:48:08,140 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0d751a622496,45911,1731847661230
2024-11-17T12:48:08,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34727-0x1014a1fb9610001, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T12:48:08,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34727-0x1014a1fb9610001, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:48:08,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T12:48:08,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:48:08,198 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T12:48:08,198 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T12:48:08,198 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T12:48:08,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:48:08,199 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:48:08,199 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34727-0x1014a1fb9610001, quorum=127.0.0.1:60904, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:48:08,199 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0d751a622496,34727,1731847661375' *****
2024-11-17T12:48:08,199 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-17T12:48:08,199 INFO  [RS:0;0d751a622496:34727 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-17T12:48:08,199 INFO  [RS:0;0d751a622496:34727 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-17T12:48:08,199 INFO  [RS:0;0d751a622496:34727 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-17T12:48:08,199 INFO  [RS:0;0d751a622496:34727 {}] regionserver.HRegionServer(3091): Received CLOSE for 4570c3f095eca1b9f8b53d3b7ef1e29d
2024-11-17T12:48:08,199 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-17T12:48:08,199 INFO  [RS:0;0d751a622496:34727 {}] regionserver.HRegionServer(959): stopping server 0d751a622496,34727,1731847661375
2024-11-17T12:48:08,200 INFO  [RS:0;0d751a622496:34727 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T12:48:08,200 INFO  [RS:0;0d751a622496:34727 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0d751a622496:34727.
2024-11-17T12:48:08,200 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4570c3f095eca1b9f8b53d3b7ef1e29d, disabling compactions & flushes
2024-11-17T12:48:08,200 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1731847662915.4570c3f095eca1b9f8b53d3b7ef1e29d.
2024-11-17T12:48:08,200 DEBUG [RS:0;0d751a622496:34727 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T12:48:08,200 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731847662915.4570c3f095eca1b9f8b53d3b7ef1e29d.
2024-11-17T12:48:08,200 DEBUG [RS:0;0d751a622496:34727 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:48:08,200 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731847662915.4570c3f095eca1b9f8b53d3b7ef1e29d. after waiting 0 ms
2024-11-17T12:48:08,200 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1731847662915.4570c3f095eca1b9f8b53d3b7ef1e29d.
2024-11-17T12:48:08,200 INFO  [RS:0;0d751a622496:34727 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-17T12:48:08,200 INFO  [RS:0;0d751a622496:34727 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-17T12:48:08,200 INFO  [RS:0;0d751a622496:34727 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-17T12:48:08,200 INFO  [RS:0;0d751a622496:34727 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-17T12:48:08,201 INFO  [RS:0;0d751a622496:34727 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-17T12:48:08,201 DEBUG [RS:0;0d751a622496:34727 {}] regionserver.HRegionServer(1325): Online Regions={4570c3f095eca1b9f8b53d3b7ef1e29d=TestLogRolling-testLogRollOnPipelineRestart,,1731847662915.4570c3f095eca1b9f8b53d3b7ef1e29d., 1588230740=hbase:meta,,1.1588230740}
2024-11-17T12:48:08,201 DEBUG [RS:0;0d751a622496:34727 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 4570c3f095eca1b9f8b53d3b7ef1e29d
2024-11-17T12:48:08,201 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T12:48:08,201 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T12:48:08,201 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T12:48:08,201 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T12:48:08,201 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T12:48:08,207 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-17T12:48:08,207 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/data/default/TestLogRolling-testLogRollOnPipelineRestart/4570c3f095eca1b9f8b53d3b7ef1e29d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1
2024-11-17T12:48:08,208 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T12:48:08,208 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T12:48:08,208 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731847662915.4570c3f095eca1b9f8b53d3b7ef1e29d.
2024-11-17T12:48:08,208 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731847688201Running coprocessor pre-close hooks at 1731847688201Disabling compacts and flushes for region at 1731847688201Disabling writes for close at 1731847688201Writing region close event to WAL at 1731847688202 (+1 ms)Running coprocessor post-close hooks at 1731847688208 (+6 ms)Closed at 1731847688208
2024-11-17T12:48:08,208 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4570c3f095eca1b9f8b53d3b7ef1e29d:
Waiting for close lock at 1731847688200Running coprocessor pre-close hooks at 1731847688200Disabling compacts and flushes for region at 1731847688200Disabling writes for close at 1731847688200Writing region close event to WAL at 1731847688201 (+1 ms)Running coprocessor post-close hooks at 1731847688208 (+7 ms)Closed at 1731847688208
2024-11-17T12:48:08,208 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-17T12:48:08,208 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731847662915.4570c3f095eca1b9f8b53d3b7ef1e29d.
2024-11-17T12:48:08,284 INFO  [regionserver/0d751a622496:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T12:48:08,310 INFO  [regionserver/0d751a622496:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-17T12:48:08,310 INFO  [regionserver/0d751a622496:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-17T12:48:08,401 INFO  [RS:0;0d751a622496:34727 {}] regionserver.HRegionServer(976): stopping server 0d751a622496,34727,1731847661375; all regions closed.
2024-11-17T12:48:08,401 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:08,401 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:08,401 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:08,402 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:08,402 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:08,404 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741842_1025 (size=825)
2024-11-17T12:48:08,404 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41777 is added to blk_1073741842_1025 (size=825)
2024-11-17T12:48:08,792 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:08,798 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:09,793 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:09,798 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:10,249 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741834_1013: GenerationStamp not matched, existing replica is blk_1073741834_1010
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-17T12:48:10,479 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T12:48:10,480 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-17T12:48:10,480 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-17T12:48:10,794 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:10,799 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:11,194 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-17T12:48:11,794 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:11,799 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:12,014 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.meta.1731847662796.meta after 4001ms
2024-11-17T12:48:12,014 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/WALs/0d751a622496,34727,1731847661375/0d751a622496%2C34727%2C1731847661375.meta.1731847662796.meta to hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/oldWALs/0d751a622496%2C34727%2C1731847661375.meta.1731847662796.meta
2024-11-17T12:48:12,018 DEBUG [RS:0;0d751a622496:34727 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/oldWALs
2024-11-17T12:48:12,018 INFO  [RS:0;0d751a622496:34727 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0d751a622496%2C34727%2C1731847661375.meta:.meta(num 1731847688006)
2024-11-17T12:48:12,018 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:12,018 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:12,018 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:12,018 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:12,019 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:12,020 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41777 is added to blk_1073741840_1023 (size=1162)
2024-11-17T12:48:12,021 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741840_1023 (size=1162)
2024-11-17T12:48:12,026 DEBUG [RS:0;0d751a622496:34727 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/oldWALs
2024-11-17T12:48:12,027 INFO  [RS:0;0d751a622496:34727 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0d751a622496%2C34727%2C1731847661375:(num 1731847687953)
2024-11-17T12:48:12,027 DEBUG [RS:0;0d751a622496:34727 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:48:12,027 INFO  [RS:0;0d751a622496:34727 {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T12:48:12,027 INFO  [RS:0;0d751a622496:34727 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T12:48:12,027 INFO  [RS:0;0d751a622496:34727 {}] hbase.ChoreService(370): Chore service for: regionserver/0d751a622496:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-17T12:48:12,027 INFO  [RS:0;0d751a622496:34727 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T12:48:12,027 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T12:48:12,027 INFO  [RS:0;0d751a622496:34727 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34727
2024-11-17T12:48:12,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34727-0x1014a1fb9610001, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0d751a622496,34727,1731847661375
2024-11-17T12:48:12,080 INFO  [RS:0;0d751a622496:34727 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T12:48:12,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T12:48:12,089 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0d751a622496,34727,1731847661375]
2024-11-17T12:48:12,097 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0d751a622496,34727,1731847661375 already deleted, retry=false
2024-11-17T12:48:12,097 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0d751a622496,34727,1731847661375 expired; onlineServers=0
2024-11-17T12:48:12,097 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0d751a622496,45911,1731847661230' *****
2024-11-17T12:48:12,097 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-17T12:48:12,097 INFO  [M:0;0d751a622496:45911 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T12:48:12,097 INFO  [M:0;0d751a622496:45911 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T12:48:12,097 DEBUG [M:0;0d751a622496:45911 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-17T12:48:12,097 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-17T12:48:12,097 DEBUG [M:0;0d751a622496:45911 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-17T12:48:12,097 DEBUG [master/0d751a622496:0:becomeActiveMaster-HFileCleaner.large.0-1731847662150 {}] cleaner.HFileCleaner(306): Exit Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.large.0-1731847662150,5,FailOnTimeoutGroup]
2024-11-17T12:48:12,097 DEBUG [master/0d751a622496:0:becomeActiveMaster-HFileCleaner.small.0-1731847662151 {}] cleaner.HFileCleaner(306): Exit Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.small.0-1731847662151,5,FailOnTimeoutGroup]
2024-11-17T12:48:12,097 INFO  [M:0;0d751a622496:45911 {}] hbase.ChoreService(370): Chore service for: master/0d751a622496:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-17T12:48:12,098 INFO  [M:0;0d751a622496:45911 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T12:48:12,098 DEBUG [M:0;0d751a622496:45911 {}] master.HMaster(1795): Stopping service threads
2024-11-17T12:48:12,098 INFO  [M:0;0d751a622496:45911 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-17T12:48:12,098 INFO  [M:0;0d751a622496:45911 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T12:48:12,098 INFO  [M:0;0d751a622496:45911 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-17T12:48:12,098 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-17T12:48:12,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-17T12:48:12,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:48:12,105 DEBUG [M:0;0d751a622496:45911 {}] zookeeper.ZKUtil(347): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-17T12:48:12,105 WARN  [M:0;0d751a622496:45911 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-17T12:48:12,106 INFO  [M:0;0d751a622496:45911 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/.lastflushedseqids
2024-11-17T12:48:12,112 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41777 is added to blk_1073741846_1030 (size=111)
2024-11-17T12:48:12,112 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741846_1030 (size=111)
2024-11-17T12:48:12,113 INFO  [M:0;0d751a622496:45911 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-17T12:48:12,113 INFO  [M:0;0d751a622496:45911 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-17T12:48:12,113 DEBUG [M:0;0d751a622496:45911 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T12:48:12,113 INFO  [M:0;0d751a622496:45911 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:48:12,113 DEBUG [M:0;0d751a622496:45911 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:48:12,113 DEBUG [M:0;0d751a622496:45911 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T12:48:12,113 DEBUG [M:0;0d751a622496:45911 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:48:12,113 INFO  [M:0;0d751a622496:45911 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.18 KB heapSize=29.16 KB
2024-11-17T12:48:12,114 ERROR [FSHLog-0-hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData-prefix:0d751a622496,45911,1731847661230 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36187,DS-40ff578d-b4b3-4910-8233-f15f8ec9c022,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:48:12,114 WARN  [FSHLog-0-hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData-prefix:0d751a622496,45911,1731847661230 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36187,DS-40ff578d-b4b3-4910-8233-f15f8ec9c022,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:48:12,114 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog 0d751a622496%2C45911%2C1731847661230:(num 1731847661914) roll requested
2024-11-17T12:48:12,114 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C45911%2C1731847661230.1731847692114
2024-11-17T12:48:12,122 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:12,122 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:12,122 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:12,122 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:12,122 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:12,122 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/WALs/0d751a622496,45911,1731847661230/0d751a622496%2C45911%2C1731847661230.1731847661914 with entries=53, filesize=26.63 KB; new WAL /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/WALs/0d751a622496,45911,1731847661230/0d751a622496%2C45911%2C1731847661230.1731847692114
2024-11-17T12:48:12,123 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36187,DS-40ff578d-b4b3-4910-8233-f15f8ec9c022,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:48:12,123 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36187,DS-40ff578d-b4b3-4910-8233-f15f8ec9c022,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-17T12:48:12,123 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/WALs/0d751a622496,45911,1731847661230/0d751a622496%2C45911%2C1731847661230.1731847661914
2024-11-17T12:48:12,123 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41483:41483),(127.0.0.1/127.0.0.1:46123:46123)]
2024-11-17T12:48:12,123 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/WALs/0d751a622496,45911,1731847661230/0d751a622496%2C45911%2C1731847661230.1731847661914 is not closed yet, will try archiving it next time
2024-11-17T12:48:12,123 WARN  [IPC Server handler 3 on default port 42599 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/WALs/0d751a622496,45911,1731847661230/0d751a622496%2C45911%2C1731847661230.1731847661914 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1015
2024-11-17T12:48:12,123 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/WALs/0d751a622496,45911,1731847661230/0d751a622496%2C45911%2C1731847661230.1731847661914 after 0ms
2024-11-17T12:48:12,141 DEBUG [M:0;0d751a622496:45911 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/701e37a7d17142e98f18840138a4e86f is 82, key is hbase:meta,,1/info:regioninfo/1731847662827/Put/seqid=0
2024-11-17T12:48:12,146 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41777 is added to blk_1073741848_1033 (size=5672)
2024-11-17T12:48:12,146 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741848_1033 (size=5672)
2024-11-17T12:48:12,146 INFO  [M:0;0d751a622496:45911 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/701e37a7d17142e98f18840138a4e86f
2024-11-17T12:48:12,166 DEBUG [M:0;0d751a622496:45911 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2764b0558ce1439189e79a25e6c1c65a is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731847663300/Put/seqid=0
2024-11-17T12:48:12,171 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741849_1034 (size=6119)
2024-11-17T12:48:12,171 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41777 is added to blk_1073741849_1034 (size=6119)
2024-11-17T12:48:12,172 INFO  [M:0;0d751a622496:45911 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.58 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2764b0558ce1439189e79a25e6c1c65a
2024-11-17T12:48:12,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34727-0x1014a1fb9610001, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:48:12,189 INFO  [RS:0;0d751a622496:34727 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T12:48:12,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34727-0x1014a1fb9610001, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:48:12,189 INFO  [RS:0;0d751a622496:34727 {}] regionserver.HRegionServer(1031): Exiting; stopping=0d751a622496,34727,1731847661375; zookeeper connection closed.
2024-11-17T12:48:12,189 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6496e96f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6496e96f
2024-11-17T12:48:12,189 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-17T12:48:12,194 DEBUG [M:0;0d751a622496:45911 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/aff3411af62f41c29a380fb8e5efb7cc is 69, key is 0d751a622496,34727,1731847661375/rs:state/1731847662260/Put/seqid=0
2024-11-17T12:48:12,199 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741850_1035 (size=5156)
2024-11-17T12:48:12,200 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41777 is added to blk_1073741850_1035 (size=5156)
2024-11-17T12:48:12,200 INFO  [M:0;0d751a622496:45911 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/aff3411af62f41c29a380fb8e5efb7cc
2024-11-17T12:48:12,223 DEBUG [M:0;0d751a622496:45911 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/3d925183cd1240d0abbdb82df02984c9 is 52, key is load_balancer_on/state:d/1731847662910/Put/seqid=0
2024-11-17T12:48:12,227 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741851_1036 (size=5056)
2024-11-17T12:48:12,228 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41777 is added to blk_1073741851_1036 (size=5056)
2024-11-17T12:48:12,228 INFO  [M:0;0d751a622496:45911 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/3d925183cd1240d0abbdb82df02984c9
2024-11-17T12:48:12,233 DEBUG [M:0;0d751a622496:45911 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/701e37a7d17142e98f18840138a4e86f as hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/701e37a7d17142e98f18840138a4e86f
2024-11-17T12:48:12,239 INFO  [M:0;0d751a622496:45911 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/701e37a7d17142e98f18840138a4e86f, entries=8, sequenceid=56, filesize=5.5 K
2024-11-17T12:48:12,240 DEBUG [M:0;0d751a622496:45911 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2764b0558ce1439189e79a25e6c1c65a as hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2764b0558ce1439189e79a25e6c1c65a
2024-11-17T12:48:12,246 INFO  [M:0;0d751a622496:45911 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2764b0558ce1439189e79a25e6c1c65a, entries=6, sequenceid=56, filesize=6.0 K
2024-11-17T12:48:12,247 DEBUG [M:0;0d751a622496:45911 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/aff3411af62f41c29a380fb8e5efb7cc as hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/aff3411af62f41c29a380fb8e5efb7cc
2024-11-17T12:48:12,253 INFO  [M:0;0d751a622496:45911 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/aff3411af62f41c29a380fb8e5efb7cc, entries=1, sequenceid=56, filesize=5.0 K
2024-11-17T12:48:12,254 DEBUG [M:0;0d751a622496:45911 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/3d925183cd1240d0abbdb82df02984c9 as hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/3d925183cd1240d0abbdb82df02984c9
2024-11-17T12:48:12,259 INFO  [M:0;0d751a622496:45911 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/3d925183cd1240d0abbdb82df02984c9, entries=1, sequenceid=56, filesize=4.9 K
2024-11-17T12:48:12,261 INFO  [M:0;0d751a622496:45911 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.18 KB/23738, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 147ms, sequenceid=56, compaction requested=false
2024-11-17T12:48:12,262 INFO  [M:0;0d751a622496:45911 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:48:12,262 DEBUG [M:0;0d751a622496:45911 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731847692113Disabling compacts and flushes for region at 1731847692113Disabling writes for close at 1731847692113Obtaining lock to block concurrent updates at 1731847692113Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731847692113Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23738, getHeapSize=29800, getOffHeapSize=0, getCellsCount=67 at 1731847692114 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731847692124 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731847692124Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731847692141 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731847692141Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731847692151 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731847692166 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731847692166Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731847692176 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731847692193 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731847692193Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731847692205 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731847692222 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731847692222Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f2612f4: reopening flushed file at 1731847692233 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72b4fc2a: reopening flushed file at 1731847692239 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@267b8bbb: reopening flushed file at 1731847692246 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11bd0a90: reopening flushed file at 1731847692253 (+7 ms)Finished flush of dataSize ~23.18 KB/23738, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 147ms, sequenceid=56, compaction requested=false at 1731847692261 (+8 ms)Writing region close event to WAL at 1731847692262 (+1 ms)Closed at 1731847692262
2024-11-17T12:48:12,263 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:12,263 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:12,263 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:12,263 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:12,263 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:12,265 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741847_1031 (size=757)
2024-11-17T12:48:12,265 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41777 is added to blk_1073741847_1031 (size=757)
2024-11-17T12:48:12,795 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:12,800 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:13,209 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,209 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,234 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,234 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,235 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,235 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,235 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,235 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,240 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,240 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,240 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,243 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,248 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,248 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,751 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-17T12:48:13,753 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,753 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,753 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,754 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,778 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,779 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,779 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,779 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,779 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,780 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,783 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,784 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,784 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,787 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:13,796 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:13,800 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:14,796 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:14,801 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:15,797 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:15,801 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:16,124 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/WALs/0d751a622496,45911,1731847661230/0d751a622496%2C45911%2C1731847661230.1731847661914 after 4001ms
2024-11-17T12:48:16,125 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/WALs/0d751a622496,45911,1731847661230/0d751a622496%2C45911%2C1731847661230.1731847661914 to hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/oldWALs/0d751a622496%2C45911%2C1731847661230.1731847661914
2024-11-17T12:48:16,127 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/MasterData/oldWALs/0d751a622496%2C45911%2C1731847661230.1731847661914 to hdfs://localhost:42599/user/jenkins/test-data/c9c93a39-5266-5fca-a7b1-25d1e91ae0eb/oldWALs/0d751a622496%2C45911%2C1731847661230.1731847661914$masterlocalwal$
2024-11-17T12:48:16,127 INFO  [M:0;0d751a622496:45911 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-17T12:48:16,127 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T12:48:16,128 INFO  [M:0;0d751a622496:45911 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45911
2024-11-17T12:48:16,128 INFO  [M:0;0d751a622496:45911 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T12:48:16,249 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741830_1015: GenerationStamp not matched, existing replica is blk_1073741830_1006
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-17T12:48:16,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:48:16,280 INFO  [M:0;0d751a622496:45911 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T12:48:16,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45911-0x1014a1fb9610000, quorum=127.0.0.1:60904, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:48:16,315 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@40c9d28{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:48:16,316 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@53bcf6e3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:48:16,316 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:48:16,316 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@78489204{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:48:16,317 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@157f7042{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/hadoop.log.dir/,STOPPED}
2024-11-17T12:48:16,320 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T12:48:16,320 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T12:48:16,320 WARN  [BP-1897123262-172.17.0.2-1731847659391 heartbeating to localhost/127.0.0.1:42599 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T12:48:16,320 WARN  [BP-1897123262-172.17.0.2-1731847659391 heartbeating to localhost/127.0.0.1:42599 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1897123262-172.17.0.2-1731847659391 (Datanode Uuid 4d688259-a8bb-437b-95bd-a86e6fe826d7) service to localhost/127.0.0.1:42599
2024-11-17T12:48:16,321 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/cluster_b93fc69f-3ebf-86a6-e261-c77f3932f38f/data/data3/current/BP-1897123262-172.17.0.2-1731847659391 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:48:16,321 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/cluster_b93fc69f-3ebf-86a6-e261-c77f3932f38f/data/data4/current/BP-1897123262-172.17.0.2-1731847659391 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:48:16,321 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T12:48:16,325 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2da9d855{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:48:16,325 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4e60361d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:48:16,325 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:48:16,325 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5d459d02{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:48:16,325 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6ba84765{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/hadoop.log.dir/,STOPPED}
2024-11-17T12:48:16,326 WARN  [BP-1897123262-172.17.0.2-1731847659391 heartbeating to localhost/127.0.0.1:42599 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T12:48:16,326 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T12:48:16,327 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T12:48:16,327 WARN  [BP-1897123262-172.17.0.2-1731847659391 heartbeating to localhost/127.0.0.1:42599 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1897123262-172.17.0.2-1731847659391 (Datanode Uuid 75852f8c-5eae-4c94-85d9-124ca4775ed8) service to localhost/127.0.0.1:42599
2024-11-17T12:48:16,327 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/cluster_b93fc69f-3ebf-86a6-e261-c77f3932f38f/data/data1/current/BP-1897123262-172.17.0.2-1731847659391 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:48:16,327 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/cluster_b93fc69f-3ebf-86a6-e261-c77f3932f38f/data/data2/current/BP-1897123262-172.17.0.2-1731847659391 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:48:16,328 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T12:48:16,334 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6a724f0c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T12:48:16,334 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@17be9294{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:48:16,335 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:48:16,335 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@33392a77{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:48:16,335 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@49d720be{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/hadoop.log.dir/,STOPPED}
2024-11-17T12:48:16,341 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-17T12:48:16,363 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-17T12:48:16,373 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=179 (was 154)
Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42599
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:42599
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:42599 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-33-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42599
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:42599 from jenkins.hfs.4
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:42599 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42599
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:42599
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=457 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=220 (was 197) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5306 (was 5557)
2024-11-17T12:48:16,382 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=179, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=220, ProcessCount=11, AvailableMemoryMB=5306
2024-11-17T12:48:16,382 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-17T12:48:16,382 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/hadoop.log.dir so I do NOT create it in target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570
2024-11-17T12:48:16,383 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/70156d81-2d61-0591-294c-e0e8cf0a6ab5/hadoop.tmp.dir so I do NOT create it in target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570
2024-11-17T12:48:16,383 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/cluster_389f1386-0ea4-b3cd-9518-5fd758751bda, deleteOnExit=true
2024-11-17T12:48:16,383 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-17T12:48:16,383 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/test.cache.data in system properties and HBase conf
2024-11-17T12:48:16,383 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/hadoop.tmp.dir in system properties and HBase conf
2024-11-17T12:48:16,383 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/hadoop.log.dir in system properties and HBase conf
2024-11-17T12:48:16,383 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-17T12:48:16,384 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-17T12:48:16,384 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-17T12:48:16,384 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-17T12:48:16,384 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-17T12:48:16,384 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-17T12:48:16,384 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-17T12:48:16,384 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T12:48:16,385 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-17T12:48:16,385 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-17T12:48:16,385 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T12:48:16,385 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T12:48:16,385 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-17T12:48:16,385 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/nfs.dump.dir in system properties and HBase conf
2024-11-17T12:48:16,385 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/java.io.tmpdir in system properties and HBase conf
2024-11-17T12:48:16,385 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T12:48:16,385 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-17T12:48:16,386 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-17T12:48:16,403 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T12:48:16,693 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:48:16,697 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:48:16,699 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:48:16,699 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:48:16,699 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T12:48:16,699 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:48:16,700 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1c41fb6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:48:16,700 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b8edabe{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:48:16,798 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:16,802 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:16,804 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4d36967f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/java.io.tmpdir/jetty-localhost-37691-hadoop-hdfs-3_4_1-tests_jar-_-any-10525765240340496278/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T12:48:16,805 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4d389c80{HTTP/1.1, (http/1.1)}{localhost:37691}
2024-11-17T12:48:16,805 INFO  [Time-limited test {}] server.Server(415): Started @192047ms
2024-11-17T12:48:16,817 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T12:48:17,041 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:48:17,045 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:48:17,045 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:48:17,045 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:48:17,045 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T12:48:17,046 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@10c2896a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:48:17,046 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4bd1d692{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:48:17,149 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2526c219{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/java.io.tmpdir/jetty-localhost-45505-hadoop-hdfs-3_4_1-tests_jar-_-any-2312179997230876259/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:48:17,149 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@74e6f5d9{HTTP/1.1, (http/1.1)}{localhost:45505}
2024-11-17T12:48:17,149 INFO  [Time-limited test {}] server.Server(415): Started @192391ms
2024-11-17T12:48:17,150 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T12:48:17,177 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:48:17,181 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:48:17,181 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:48:17,181 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:48:17,181 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T12:48:17,182 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5894e22d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:48:17,182 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41bc1801{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:48:17,297 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5cdf1f1c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/java.io.tmpdir/jetty-localhost-44285-hadoop-hdfs-3_4_1-tests_jar-_-any-5137878555457212487/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:48:17,297 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@51e1410e{HTTP/1.1, (http/1.1)}{localhost:44285}
2024-11-17T12:48:17,297 INFO  [Time-limited test {}] server.Server(415): Started @192539ms
2024-11-17T12:48:17,298 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T12:48:17,799 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:17,802 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:17,883 WARN  [Thread-1650 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/cluster_389f1386-0ea4-b3cd-9518-5fd758751bda/data/data1/current/BP-1721540678-172.17.0.2-1731847696414/current, will proceed with Du for space computation calculation, 
2024-11-17T12:48:17,883 WARN  [Thread-1651 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/cluster_389f1386-0ea4-b3cd-9518-5fd758751bda/data/data2/current/BP-1721540678-172.17.0.2-1731847696414/current, will proceed with Du for space computation calculation, 
2024-11-17T12:48:17,901 WARN  [Thread-1614 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T12:48:17,904 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5988f72d69d823e3 with lease ID 0x88ac0090afbc16e1: Processing first storage report for DS-9696e41e-d99c-49b7-822c-b65e58b9884d from datanode DatanodeRegistration(127.0.0.1:38897, datanodeUuid=1088402a-af37-4455-98d9-ff57b6e5ec20, infoPort=45821, infoSecurePort=0, ipcPort=39313, storageInfo=lv=-57;cid=testClusterID;nsid=392821933;c=1731847696414)
2024-11-17T12:48:17,904 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5988f72d69d823e3 with lease ID 0x88ac0090afbc16e1: from storage DS-9696e41e-d99c-49b7-822c-b65e58b9884d node DatanodeRegistration(127.0.0.1:38897, datanodeUuid=1088402a-af37-4455-98d9-ff57b6e5ec20, infoPort=45821, infoSecurePort=0, ipcPort=39313, storageInfo=lv=-57;cid=testClusterID;nsid=392821933;c=1731847696414), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:48:17,904 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5988f72d69d823e3 with lease ID 0x88ac0090afbc16e1: Processing first storage report for DS-344942a5-e7f3-48b8-b8cf-dfcdde2db6d7 from datanode DatanodeRegistration(127.0.0.1:38897, datanodeUuid=1088402a-af37-4455-98d9-ff57b6e5ec20, infoPort=45821, infoSecurePort=0, ipcPort=39313, storageInfo=lv=-57;cid=testClusterID;nsid=392821933;c=1731847696414)
2024-11-17T12:48:17,904 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5988f72d69d823e3 with lease ID 0x88ac0090afbc16e1: from storage DS-344942a5-e7f3-48b8-b8cf-dfcdde2db6d7 node DatanodeRegistration(127.0.0.1:38897, datanodeUuid=1088402a-af37-4455-98d9-ff57b6e5ec20, infoPort=45821, infoSecurePort=0, ipcPort=39313, storageInfo=lv=-57;cid=testClusterID;nsid=392821933;c=1731847696414), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:48:18,058 WARN  [Thread-1661 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/cluster_389f1386-0ea4-b3cd-9518-5fd758751bda/data/data3/current/BP-1721540678-172.17.0.2-1731847696414/current, will proceed with Du for space computation calculation, 
2024-11-17T12:48:18,058 WARN  [Thread-1662 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/cluster_389f1386-0ea4-b3cd-9518-5fd758751bda/data/data4/current/BP-1721540678-172.17.0.2-1731847696414/current, will proceed with Du for space computation calculation, 
2024-11-17T12:48:18,077 WARN  [Thread-1637 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T12:48:18,079 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc03af6814ffd5c89 with lease ID 0x88ac0090afbc16e2: Processing first storage report for DS-48faef62-17ed-4185-9542-9433eb03d102 from datanode DatanodeRegistration(127.0.0.1:32911, datanodeUuid=0a8dfbd1-2528-452a-bcd4-4eaadcb0c577, infoPort=46531, infoSecurePort=0, ipcPort=46133, storageInfo=lv=-57;cid=testClusterID;nsid=392821933;c=1731847696414)
2024-11-17T12:48:18,079 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc03af6814ffd5c89 with lease ID 0x88ac0090afbc16e2: from storage DS-48faef62-17ed-4185-9542-9433eb03d102 node DatanodeRegistration(127.0.0.1:32911, datanodeUuid=0a8dfbd1-2528-452a-bcd4-4eaadcb0c577, infoPort=46531, infoSecurePort=0, ipcPort=46133, storageInfo=lv=-57;cid=testClusterID;nsid=392821933;c=1731847696414), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:48:18,079 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc03af6814ffd5c89 with lease ID 0x88ac0090afbc16e2: Processing first storage report for DS-6a2db01e-7bf3-4dc2-b2e2-96dabb11ff70 from datanode DatanodeRegistration(127.0.0.1:32911, datanodeUuid=0a8dfbd1-2528-452a-bcd4-4eaadcb0c577, infoPort=46531, infoSecurePort=0, ipcPort=46133, storageInfo=lv=-57;cid=testClusterID;nsid=392821933;c=1731847696414)
2024-11-17T12:48:18,079 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc03af6814ffd5c89 with lease ID 0x88ac0090afbc16e2: from storage DS-6a2db01e-7bf3-4dc2-b2e2-96dabb11ff70 node DatanodeRegistration(127.0.0.1:32911, datanodeUuid=0a8dfbd1-2528-452a-bcd4-4eaadcb0c577, infoPort=46531, infoSecurePort=0, ipcPort=46133, storageInfo=lv=-57;cid=testClusterID;nsid=392821933;c=1731847696414), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:48:18,125 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570
2024-11-17T12:48:18,128 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/cluster_389f1386-0ea4-b3cd-9518-5fd758751bda/zookeeper_0, clientPort=56716, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/cluster_389f1386-0ea4-b3cd-9518-5fd758751bda/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/cluster_389f1386-0ea4-b3cd-9518-5fd758751bda/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-17T12:48:18,129 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56716
2024-11-17T12:48:18,129 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:48:18,131 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:48:18,140 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741825_1001 (size=7)
2024-11-17T12:48:18,140 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741825_1001 (size=7)
2024-11-17T12:48:18,142 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226 with version=8
2024-11-17T12:48:18,142 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/hbase-staging
2024-11-17T12:48:18,144 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/0d751a622496:0 server-side Connection retries=45
2024-11-17T12:48:18,144 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:48:18,144 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T12:48:18,144 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T12:48:18,144 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:48:18,144 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T12:48:18,144 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-17T12:48:18,144 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T12:48:18,145 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39777
2024-11-17T12:48:18,147 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39777 connecting to ZooKeeper ensemble=127.0.0.1:56716
2024-11-17T12:48:18,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:397770x0, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T12:48:18,209 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39777-0x1014a2049940000 connected
2024-11-17T12:48:18,284 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:48:18,286 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:48:18,289 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:48:18,289 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226, hbase.cluster.distributed=false
2024-11-17T12:48:18,291 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T12:48:18,292 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39777
2024-11-17T12:48:18,292 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39777
2024-11-17T12:48:18,292 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39777
2024-11-17T12:48:18,292 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39777
2024-11-17T12:48:18,293 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39777
2024-11-17T12:48:18,309 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/0d751a622496:0 server-side Connection retries=45
2024-11-17T12:48:18,309 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:48:18,309 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T12:48:18,309 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T12:48:18,309 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:48:18,309 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T12:48:18,309 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-17T12:48:18,309 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T12:48:18,310 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34965
2024-11-17T12:48:18,312 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34965 connecting to ZooKeeper ensemble=127.0.0.1:56716
2024-11-17T12:48:18,312 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:48:18,314 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:48:18,326 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:349650x0, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T12:48:18,326 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:349650x0, quorum=127.0.0.1:56716, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:48:18,326 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34965-0x1014a2049940001 connected
2024-11-17T12:48:18,326 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-17T12:48:18,327 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-17T12:48:18,327 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34965-0x1014a2049940001, quorum=127.0.0.1:56716, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-17T12:48:18,328 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34965-0x1014a2049940001, quorum=127.0.0.1:56716, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T12:48:18,329 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34965
2024-11-17T12:48:18,329 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34965
2024-11-17T12:48:18,329 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34965
2024-11-17T12:48:18,330 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34965
2024-11-17T12:48:18,330 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34965
2024-11-17T12:48:18,345 DEBUG [M:0;0d751a622496:39777 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0d751a622496:39777
2024-11-17T12:48:18,345 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0d751a622496,39777,1731847698143
2024-11-17T12:48:18,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:48:18,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34965-0x1014a2049940001, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:48:18,355 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0d751a622496,39777,1731847698143
2024-11-17T12:48:18,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34965-0x1014a2049940001, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-17T12:48:18,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:48:18,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34965-0x1014a2049940001, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:48:18,364 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-17T12:48:18,364 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0d751a622496,39777,1731847698143 from backup master directory
2024-11-17T12:48:18,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34965-0x1014a2049940001, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:48:18,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0d751a622496,39777,1731847698143
2024-11-17T12:48:18,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:48:18,372 WARN  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T12:48:18,372 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0d751a622496,39777,1731847698143
2024-11-17T12:48:18,380 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/hbase.id] with ID: 4a1b708b-691f-4bb1-8d5d-95cfeec94f4e
2024-11-17T12:48:18,380 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/.tmp/hbase.id
2024-11-17T12:48:18,385 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741826_1002 (size=42)
2024-11-17T12:48:18,386 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741826_1002 (size=42)
2024-11-17T12:48:18,387 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/.tmp/hbase.id]:[hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/hbase.id]
2024-11-17T12:48:18,397 INFO  [master/0d751a622496:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:48:18,397 INFO  [master/0d751a622496:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-17T12:48:18,400 INFO  [master/0d751a622496:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms.
2024-11-17T12:48:18,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:48:18,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34965-0x1014a2049940001, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:48:18,414 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741827_1003 (size=196)
2024-11-17T12:48:18,414 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741827_1003 (size=196)
2024-11-17T12:48:18,415 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-17T12:48:18,416 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-17T12:48:18,416 INFO  [master/0d751a622496:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T12:48:18,424 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741828_1004 (size=1189)
2024-11-17T12:48:18,424 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741828_1004 (size=1189)
2024-11-17T12:48:18,425 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/data/master/store
2024-11-17T12:48:18,431 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741829_1005 (size=34)
2024-11-17T12:48:18,432 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741829_1005 (size=34)
2024-11-17T12:48:18,432 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:48:18,432 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T12:48:18,432 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:48:18,432 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:48:18,432 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T12:48:18,432 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:48:18,432 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:48:18,432 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731847698432Disabling compacts and flushes for region at 1731847698432Disabling writes for close at 1731847698432Writing region close event to WAL at 1731847698432Closed at 1731847698432
2024-11-17T12:48:18,433 WARN  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/data/master/store/.initializing
2024-11-17T12:48:18,433 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/WALs/0d751a622496,39777,1731847698143
2024-11-17T12:48:18,435 INFO  [master/0d751a622496:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C39777%2C1731847698143, suffix=, logDir=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/WALs/0d751a622496,39777,1731847698143, archiveDir=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/oldWALs, maxLogs=10
2024-11-17T12:48:18,436 INFO  [master/0d751a622496:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C39777%2C1731847698143.1731847698436
2024-11-17T12:48:18,440 INFO  [master/0d751a622496:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/WALs/0d751a622496,39777,1731847698143/0d751a622496%2C39777%2C1731847698143.1731847698436
2024-11-17T12:48:18,441 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45821:45821),(127.0.0.1/127.0.0.1:46531:46531)]
2024-11-17T12:48:18,447 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-17T12:48:18,447 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:48:18,447 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:48:18,447 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:48:18,449 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:48:18,450 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-17T12:48:18,450 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:48:18,450 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:48:18,451 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:48:18,452 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-17T12:48:18,452 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:48:18,452 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:48:18,452 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:48:18,453 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-17T12:48:18,454 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:48:18,454 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:48:18,454 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:48:18,455 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-17T12:48:18,455 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:48:18,456 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:48:18,456 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:48:18,457 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:48:18,457 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:48:18,458 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:48:18,458 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:48:18,459 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-17T12:48:18,460 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:48:18,462 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T12:48:18,463 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=738587, jitterRate=-0.060838714241981506}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-17T12:48:18,464 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731847698447Initializing all the Stores at 1731847698448 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847698448Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847698448Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847698448Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847698449 (+1 ms)Cleaning up temporary data from old regions at 1731847698458 (+9 ms)Region opened successfully at 1731847698464 (+6 ms)
2024-11-17T12:48:18,464 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-17T12:48:18,467 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e4cd767, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0d751a622496/172.17.0.2:0
2024-11-17T12:48:18,468 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-17T12:48:18,468 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-17T12:48:18,468 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-17T12:48:18,468 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-17T12:48:18,469 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-17T12:48:18,469 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-17T12:48:18,469 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-17T12:48:18,471 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-17T12:48:18,472 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-17T12:48:18,500 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-17T12:48:18,501 INFO  [master/0d751a622496:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-17T12:48:18,502 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-17T12:48:18,509 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-17T12:48:18,509 INFO  [master/0d751a622496:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-17T12:48:18,510 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-17T12:48:18,517 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-17T12:48:18,518 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-17T12:48:18,525 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-17T12:48:18,527 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-17T12:48:18,538 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-17T12:48:18,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T12:48:18,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34965-0x1014a2049940001, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T12:48:18,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34965-0x1014a2049940001, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:48:18,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:48:18,547 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0d751a622496,39777,1731847698143, sessionid=0x1014a2049940000, setting cluster-up flag (Was=false)
2024-11-17T12:48:18,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:48:18,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34965-0x1014a2049940001, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:48:18,588 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-17T12:48:18,589 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0d751a622496,39777,1731847698143
2024-11-17T12:48:18,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34965-0x1014a2049940001, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:48:18,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:48:18,634 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-17T12:48:18,635 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0d751a622496,39777,1731847698143
2024-11-17T12:48:18,636 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-17T12:48:18,638 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-17T12:48:18,638 INFO  [master/0d751a622496:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-17T12:48:18,638 INFO  [master/0d751a622496:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-17T12:48:18,638 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0d751a622496,39777,1731847698143
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-17T12:48:18,640 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:48:18,640 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:48:18,640 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:48:18,640 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:48:18,640 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0d751a622496:0, corePoolSize=10, maxPoolSize=10
2024-11-17T12:48:18,640 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:48:18,640 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0d751a622496:0, corePoolSize=2, maxPoolSize=2
2024-11-17T12:48:18,640 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:48:18,641 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731847728641
2024-11-17T12:48:18,641 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-17T12:48:18,642 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-17T12:48:18,642 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-17T12:48:18,642 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-17T12:48:18,642 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-17T12:48:18,642 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-17T12:48:18,647 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T12:48:18,647 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-17T12:48:18,647 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T12:48:18,648 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-17T12:48:18,648 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-17T12:48:18,648 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-17T12:48:18,648 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:48:18,648 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-17T12:48:18,651 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-17T12:48:18,652 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-17T12:48:18,652 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.large.0-1731847698652,5,FailOnTimeoutGroup]
2024-11-17T12:48:18,652 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.small.0-1731847698652,5,FailOnTimeoutGroup]
2024-11-17T12:48:18,652 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T12:48:18,652 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-17T12:48:18,652 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-17T12:48:18,652 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-17T12:48:18,656 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741831_1007 (size=1321)
2024-11-17T12:48:18,656 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741831_1007 (size=1321)
2024-11-17T12:48:18,657 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-17T12:48:18,657 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226
2024-11-17T12:48:18,663 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741832_1008 (size=32)
2024-11-17T12:48:18,664 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741832_1008 (size=32)
2024-11-17T12:48:18,732 INFO  [RS:0;0d751a622496:34965 {}] regionserver.HRegionServer(746): ClusterId : 4a1b708b-691f-4bb1-8d5d-95cfeec94f4e
2024-11-17T12:48:18,732 DEBUG [RS:0;0d751a622496:34965 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-17T12:48:18,748 DEBUG [RS:0;0d751a622496:34965 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-17T12:48:18,748 DEBUG [RS:0;0d751a622496:34965 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-17T12:48:18,760 DEBUG [RS:0;0d751a622496:34965 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-17T12:48:18,761 DEBUG [RS:0;0d751a622496:34965 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@574177e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0d751a622496/172.17.0.2:0
2024-11-17T12:48:18,778 DEBUG [RS:0;0d751a622496:34965 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0d751a622496:34965
2024-11-17T12:48:18,779 INFO  [RS:0;0d751a622496:34965 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-17T12:48:18,779 INFO  [RS:0;0d751a622496:34965 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-17T12:48:18,779 DEBUG [RS:0;0d751a622496:34965 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-17T12:48:18,779 INFO  [RS:0;0d751a622496:34965 {}] regionserver.HRegionServer(2659): reportForDuty to master=0d751a622496,39777,1731847698143 with port=34965, startcode=1731847698308
2024-11-17T12:48:18,780 DEBUG [RS:0;0d751a622496:34965 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-17T12:48:18,782 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47503, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-11-17T12:48:18,782 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39777 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0d751a622496,34965,1731847698308
2024-11-17T12:48:18,782 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39777 {}] master.ServerManager(517): Registering regionserver=0d751a622496,34965,1731847698308
2024-11-17T12:48:18,784 DEBUG [RS:0;0d751a622496:34965 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226
2024-11-17T12:48:18,784 DEBUG [RS:0;0d751a622496:34965 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40479
2024-11-17T12:48:18,784 DEBUG [RS:0;0d751a622496:34965 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-17T12:48:18,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T12:48:18,792 DEBUG [RS:0;0d751a622496:34965 {}] zookeeper.ZKUtil(111): regionserver:34965-0x1014a2049940001, quorum=127.0.0.1:56716, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0d751a622496,34965,1731847698308
2024-11-17T12:48:18,792 WARN  [RS:0;0d751a622496:34965 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T12:48:18,793 INFO  [RS:0;0d751a622496:34965 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T12:48:18,793 DEBUG [RS:0;0d751a622496:34965 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/WALs/0d751a622496,34965,1731847698308
2024-11-17T12:48:18,793 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0d751a622496,34965,1731847698308]
2024-11-17T12:48:18,796 INFO  [RS:0;0d751a622496:34965 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-17T12:48:18,798 INFO  [RS:0;0d751a622496:34965 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-17T12:48:18,798 INFO  [RS:0;0d751a622496:34965 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-17T12:48:18,799 INFO  [RS:0;0d751a622496:34965 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:48:18,799 INFO  [RS:0;0d751a622496:34965 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-17T12:48:18,799 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:18,800 INFO  [RS:0;0d751a622496:34965 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-17T12:48:18,800 INFO  [RS:0;0d751a622496:34965 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-17T12:48:18,800 DEBUG [RS:0;0d751a622496:34965 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:48:18,800 DEBUG [RS:0;0d751a622496:34965 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:48:18,800 DEBUG [RS:0;0d751a622496:34965 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:48:18,800 DEBUG [RS:0;0d751a622496:34965 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:48:18,800 DEBUG [RS:0;0d751a622496:34965 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:48:18,800 DEBUG [RS:0;0d751a622496:34965 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0, corePoolSize=2, maxPoolSize=2
2024-11-17T12:48:18,800 DEBUG [RS:0;0d751a622496:34965 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:48:18,800 DEBUG [RS:0;0d751a622496:34965 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:48:18,800 DEBUG [RS:0;0d751a622496:34965 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:48:18,801 DEBUG [RS:0;0d751a622496:34965 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:48:18,801 DEBUG [RS:0;0d751a622496:34965 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:48:18,801 DEBUG [RS:0;0d751a622496:34965 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:48:18,801 DEBUG [RS:0;0d751a622496:34965 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0d751a622496:0, corePoolSize=3, maxPoolSize=3
2024-11-17T12:48:18,801 DEBUG [RS:0;0d751a622496:34965 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0, corePoolSize=3, maxPoolSize=3
2024-11-17T12:48:18,801 INFO  [RS:0;0d751a622496:34965 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T12:48:18,801 INFO  [RS:0;0d751a622496:34965 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T12:48:18,801 INFO  [RS:0;0d751a622496:34965 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:48:18,801 INFO  [RS:0;0d751a622496:34965 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-17T12:48:18,801 INFO  [RS:0;0d751a622496:34965 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-17T12:48:18,801 INFO  [RS:0;0d751a622496:34965 {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,34965,1731847698308-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T12:48:18,803 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:18,817 INFO  [RS:0;0d751a622496:34965 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-17T12:48:18,817 INFO  [RS:0;0d751a622496:34965 {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,34965,1731847698308-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:48:18,817 INFO  [RS:0;0d751a622496:34965 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:48:18,817 INFO  [RS:0;0d751a622496:34965 {}] regionserver.Replication(171): 0d751a622496,34965,1731847698308 started
2024-11-17T12:48:18,832 INFO  [RS:0;0d751a622496:34965 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:48:18,832 INFO  [RS:0;0d751a622496:34965 {}] regionserver.HRegionServer(1482): Serving as 0d751a622496,34965,1731847698308, RpcServer on 0d751a622496/172.17.0.2:34965, sessionid=0x1014a2049940001
2024-11-17T12:48:18,832 DEBUG [RS:0;0d751a622496:34965 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-17T12:48:18,832 DEBUG [RS:0;0d751a622496:34965 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0d751a622496,34965,1731847698308
2024-11-17T12:48:18,832 DEBUG [RS:0;0d751a622496:34965 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0d751a622496,34965,1731847698308'
2024-11-17T12:48:18,833 DEBUG [RS:0;0d751a622496:34965 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-17T12:48:18,833 DEBUG [RS:0;0d751a622496:34965 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-17T12:48:18,834 DEBUG [RS:0;0d751a622496:34965 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-17T12:48:18,834 DEBUG [RS:0;0d751a622496:34965 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-17T12:48:18,834 DEBUG [RS:0;0d751a622496:34965 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0d751a622496,34965,1731847698308
2024-11-17T12:48:18,834 DEBUG [RS:0;0d751a622496:34965 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0d751a622496,34965,1731847698308'
2024-11-17T12:48:18,834 DEBUG [RS:0;0d751a622496:34965 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-17T12:48:18,834 DEBUG [RS:0;0d751a622496:34965 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-17T12:48:18,834 DEBUG [RS:0;0d751a622496:34965 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-17T12:48:18,834 INFO  [RS:0;0d751a622496:34965 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-17T12:48:18,834 INFO  [RS:0;0d751a622496:34965 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-17T12:48:18,936 INFO  [RS:0;0d751a622496:34965 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C34965%2C1731847698308, suffix=, logDir=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/WALs/0d751a622496,34965,1731847698308, archiveDir=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/oldWALs, maxLogs=32
2024-11-17T12:48:18,937 INFO  [RS:0;0d751a622496:34965 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C34965%2C1731847698308.1731847698937
2024-11-17T12:48:18,942 INFO  [RS:0;0d751a622496:34965 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/WALs/0d751a622496,34965,1731847698308/0d751a622496%2C34965%2C1731847698308.1731847698937
2024-11-17T12:48:18,943 DEBUG [RS:0;0d751a622496:34965 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46531:46531),(127.0.0.1/127.0.0.1:45821:45821)]
2024-11-17T12:48:19,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:48:19,068 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T12:48:19,070 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T12:48:19,070 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:48:19,071 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:48:19,071 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T12:48:19,073 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T12:48:19,073 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:48:19,074 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:48:19,074 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T12:48:19,076 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T12:48:19,076 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:48:19,077 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:48:19,077 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T12:48:19,079 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T12:48:19,079 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:48:19,080 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:48:19,080 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T12:48:19,081 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/hbase/meta/1588230740
2024-11-17T12:48:19,081 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/hbase/meta/1588230740
2024-11-17T12:48:19,083 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T12:48:19,083 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T12:48:19,083 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T12:48:19,085 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T12:48:19,087 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T12:48:19,087 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=804497, jitterRate=0.022971585392951965}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T12:48:19,088 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731847699066Initializing all the Stores at 1731847699067 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847699067Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847699067Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847699068 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847699068Cleaning up temporary data from old regions at 1731847699083 (+15 ms)Region opened successfully at 1731847699088 (+5 ms)
2024-11-17T12:48:19,088 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T12:48:19,088 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T12:48:19,088 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T12:48:19,088 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T12:48:19,088 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T12:48:19,089 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T12:48:19,089 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731847699088Disabling compacts and flushes for region at 1731847699088Disabling writes for close at 1731847699088Writing region close event to WAL at 1731847699089 (+1 ms)Closed at 1731847699089
2024-11-17T12:48:19,090 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T12:48:19,090 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-17T12:48:19,090 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-17T12:48:19,092 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T12:48:19,093 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-17T12:48:19,243 DEBUG [0d751a622496:39777 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-17T12:48:19,243 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0d751a622496,34965,1731847698308
2024-11-17T12:48:19,245 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0d751a622496,34965,1731847698308, state=OPENING
2024-11-17T12:48:19,275 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-17T12:48:19,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34965-0x1014a2049940001, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:48:19,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:48:19,285 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T12:48:19,285 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:48:19,285 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:48:19,285 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0d751a622496,34965,1731847698308}]
2024-11-17T12:48:19,439 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-17T12:48:19,442 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44335, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-17T12:48:19,447 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-17T12:48:19,447 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T12:48:19,449 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C34965%2C1731847698308.meta, suffix=.meta, logDir=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/WALs/0d751a622496,34965,1731847698308, archiveDir=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/oldWALs, maxLogs=32
2024-11-17T12:48:19,449 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C34965%2C1731847698308.meta.1731847699449.meta
2024-11-17T12:48:19,454 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/WALs/0d751a622496,34965,1731847698308/0d751a622496%2C34965%2C1731847698308.meta.1731847699449.meta
2024-11-17T12:48:19,455 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45821:45821),(127.0.0.1/127.0.0.1:46531:46531)]
2024-11-17T12:48:19,456 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-17T12:48:19,456 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-17T12:48:19,456 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-17T12:48:19,457 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-17T12:48:19,457 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-17T12:48:19,457 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:48:19,457 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-17T12:48:19,457 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-17T12:48:19,458 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T12:48:19,459 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T12:48:19,459 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:48:19,460 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:48:19,460 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T12:48:19,461 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T12:48:19,461 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:48:19,461 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:48:19,461 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T12:48:19,462 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T12:48:19,462 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:48:19,463 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:48:19,463 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T12:48:19,463 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T12:48:19,463 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:48:19,464 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:48:19,464 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T12:48:19,465 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/hbase/meta/1588230740
2024-11-17T12:48:19,466 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/hbase/meta/1588230740
2024-11-17T12:48:19,467 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T12:48:19,467 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T12:48:19,468 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T12:48:19,469 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T12:48:19,470 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=688839, jitterRate=-0.1240968108177185}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T12:48:19,470 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-17T12:48:19,471 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731847699457Writing region info on filesystem at 1731847699457Initializing all the Stores at 1731847699458 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847699458Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847699458Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847699458Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847699458Cleaning up temporary data from old regions at 1731847699467 (+9 ms)Running coprocessor post-open hooks at 1731847699470 (+3 ms)Region opened successfully at 1731847699471 (+1 ms)
2024-11-17T12:48:19,472 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731847699439
2024-11-17T12:48:19,474 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-17T12:48:19,474 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-17T12:48:19,475 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0d751a622496,34965,1731847698308
2024-11-17T12:48:19,476 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0d751a622496,34965,1731847698308, state=OPEN
2024-11-17T12:48:19,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34965-0x1014a2049940001, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T12:48:19,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T12:48:19,515 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0d751a622496,34965,1731847698308
2024-11-17T12:48:19,515 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:48:19,515 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:48:19,518 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-17T12:48:19,518 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0d751a622496,34965,1731847698308 in 230 msec
2024-11-17T12:48:19,520 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-17T12:48:19,520 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 428 msec
2024-11-17T12:48:19,521 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T12:48:19,521 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-17T12:48:19,522 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T12:48:19,523 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0d751a622496,34965,1731847698308, seqNum=-1]
2024-11-17T12:48:19,523 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T12:48:19,524 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55389, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T12:48:19,530 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 892 msec
2024-11-17T12:48:19,530 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731847699530, completionTime=-1
2024-11-17T12:48:19,530 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-17T12:48:19,531 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-17T12:48:19,532 INFO  [master/0d751a622496:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-17T12:48:19,533 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731847759532
2024-11-17T12:48:19,533 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731847819533
2024-11-17T12:48:19,533 INFO  [master/0d751a622496:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-17T12:48:19,533 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,39777,1731847698143-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:48:19,533 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,39777,1731847698143-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:48:19,533 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,39777,1731847698143-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:48:19,533 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0d751a622496:39777, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:48:19,533 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-17T12:48:19,533 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-17T12:48:19,535 DEBUG [master/0d751a622496:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-17T12:48:19,537 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.165sec
2024-11-17T12:48:19,537 INFO  [master/0d751a622496:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-17T12:48:19,537 INFO  [master/0d751a622496:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-17T12:48:19,537 INFO  [master/0d751a622496:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-17T12:48:19,537 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-17T12:48:19,537 INFO  [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-17T12:48:19,537 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,39777,1731847698143-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T12:48:19,537 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,39777,1731847698143-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-17T12:48:19,540 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-17T12:48:19,540 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-17T12:48:19,540 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,39777,1731847698143-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:48:19,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@394b54be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T12:48:19,633 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0d751a622496,39777,-1 for getting cluster id
2024-11-17T12:48:19,633 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-17T12:48:19,635 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4a1b708b-691f-4bb1-8d5d-95cfeec94f4e'
2024-11-17T12:48:19,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-17T12:48:19,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4a1b708b-691f-4bb1-8d5d-95cfeec94f4e"

2024-11-17T12:48:19,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26cd7671, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T12:48:19,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0d751a622496,39777,-1]
2024-11-17T12:48:19,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-17T12:48:19,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:48:19,638 INFO  [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-17T12:48:19,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319a8be6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T12:48:19,640 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T12:48:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0d751a622496,34965,1731847698308, seqNum=-1]
2024-11-17T12:48:19,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T12:48:19,643 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T12:48:19,645 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0d751a622496,39777,1731847698143
2024-11-17T12:48:19,645 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:48:19,648 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-17T12:48:19,648 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-17T12:48:19,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 0d751a622496,39777,1731847698143
2024-11-17T12:48:19,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@280b747c
2024-11-17T12:48:19,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-17T12:48:19,651 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43754, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-17T12:48:19,651 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39777 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-17T12:48:19,651 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39777 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-17T12:48:19,651 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39777 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-17T12:48:19,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39777 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-17T12:48:19,654 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-17T12:48:19,654 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:48:19,654 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39777 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling"
 procId is: 4
2024-11-17T12:48:19,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39777 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-17T12:48:19,656 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-17T12:48:19,663 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741835_1011 (size=405)
2024-11-17T12:48:19,664 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741835_1011 (size=405)
2024-11-17T12:48:19,666 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 883c19deec29e3af1819980d5d6bbf71, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226
2024-11-17T12:48:19,672 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741836_1012 (size=88)
2024-11-17T12:48:19,673 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741836_1012 (size=88)
2024-11-17T12:48:19,673 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:48:19,673 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing 883c19deec29e3af1819980d5d6bbf71, disabling compactions & flushes
2024-11-17T12:48:19,673 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.
2024-11-17T12:48:19,673 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.
2024-11-17T12:48:19,673 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71. after waiting 0 ms
2024-11-17T12:48:19,674 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.
2024-11-17T12:48:19,674 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.
2024-11-17T12:48:19,674 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 883c19deec29e3af1819980d5d6bbf71:
Waiting for close lock at 1731847699673Disabling compacts and flushes for region at 1731847699673Disabling writes for close at 1731847699673Writing region close event to WAL at 1731847699674 (+1 ms)Closed at 1731847699674
2024-11-17T12:48:19,675 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-17T12:48:19,675 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1731847699675"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731847699675"}]},"ts":"1731847699675"}
2024-11-17T12:48:19,678 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-17T12:48:19,679 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-17T12:48:19,679 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731847699679"}]},"ts":"1731847699679"}
2024-11-17T12:48:19,682 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta
2024-11-17T12:48:19,682 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=883c19deec29e3af1819980d5d6bbf71, ASSIGN}]
2024-11-17T12:48:19,684 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=883c19deec29e3af1819980d5d6bbf71, ASSIGN
2024-11-17T12:48:19,685 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=883c19deec29e3af1819980d5d6bbf71, ASSIGN; state=OFFLINE, location=0d751a622496,34965,1731847698308; forceNewPlan=false, retain=false
2024-11-17T12:48:19,800 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:19,803 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:19,836 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=883c19deec29e3af1819980d5d6bbf71, regionState=OPENING, regionLocation=0d751a622496,34965,1731847698308
2024-11-17T12:48:19,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=883c19deec29e3af1819980d5d6bbf71, ASSIGN because future has completed
2024-11-17T12:48:19,839 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 883c19deec29e3af1819980d5d6bbf71, server=0d751a622496,34965,1731847698308}]
2024-11-17T12:48:19,996 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.
2024-11-17T12:48:19,997 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 883c19deec29e3af1819980d5d6bbf71, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.', STARTKEY => '', ENDKEY => ''}
2024-11-17T12:48:19,997 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling 883c19deec29e3af1819980d5d6bbf71
2024-11-17T12:48:19,997 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:48:19,997 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 883c19deec29e3af1819980d5d6bbf71
2024-11-17T12:48:19,997 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 883c19deec29e3af1819980d5d6bbf71
2024-11-17T12:48:19,998 INFO  [StoreOpener-883c19deec29e3af1819980d5d6bbf71-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 883c19deec29e3af1819980d5d6bbf71 
2024-11-17T12:48:20,000 INFO  [StoreOpener-883c19deec29e3af1819980d5d6bbf71-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 883c19deec29e3af1819980d5d6bbf71 columnFamilyName info
2024-11-17T12:48:20,000 DEBUG [StoreOpener-883c19deec29e3af1819980d5d6bbf71-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:48:20,001 INFO  [StoreOpener-883c19deec29e3af1819980d5d6bbf71-1 {}] regionserver.HStore(327): Store=883c19deec29e3af1819980d5d6bbf71/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:48:20,001 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 883c19deec29e3af1819980d5d6bbf71
2024-11-17T12:48:20,001 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71
2024-11-17T12:48:20,002 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71
2024-11-17T12:48:20,002 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 883c19deec29e3af1819980d5d6bbf71
2024-11-17T12:48:20,002 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 883c19deec29e3af1819980d5d6bbf71
2024-11-17T12:48:20,004 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 883c19deec29e3af1819980d5d6bbf71
2024-11-17T12:48:20,012 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T12:48:20,013 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 883c19deec29e3af1819980d5d6bbf71; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=867537, jitterRate=0.10313154757022858}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-17T12:48:20,013 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 883c19deec29e3af1819980d5d6bbf71
2024-11-17T12:48:20,013 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 883c19deec29e3af1819980d5d6bbf71:
Running coprocessor pre-open hook at 1731847699997Writing region info on filesystem at 1731847699997Initializing all the Stores at 1731847699998 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847699998Cleaning up temporary data from old regions at 1731847700002 (+4 ms)Running coprocessor post-open hooks at 1731847700013 (+11 ms)Region opened successfully at 1731847700013
2024-11-17T12:48:20,014 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71., pid=6, masterSystemTime=1731847699992
2024-11-17T12:48:20,017 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.
2024-11-17T12:48:20,017 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.
2024-11-17T12:48:20,018 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=883c19deec29e3af1819980d5d6bbf71, regionState=OPEN, openSeqNum=2, regionLocation=0d751a622496,34965,1731847698308
2024-11-17T12:48:20,021 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 883c19deec29e3af1819980d5d6bbf71, server=0d751a622496,34965,1731847698308 because future has completed
2024-11-17T12:48:20,025 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-17T12:48:20,025 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 883c19deec29e3af1819980d5d6bbf71, server=0d751a622496,34965,1731847698308 in 183 msec
2024-11-17T12:48:20,028 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-17T12:48:20,028 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=883c19deec29e3af1819980d5d6bbf71, ASSIGN in 343 msec
2024-11-17T12:48:20,029 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-17T12:48:20,030 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731847700029"}]},"ts":"1731847700029"}
2024-11-17T12:48:20,032 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta
2024-11-17T12:48:20,033 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-17T12:48:20,036 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 381 msec
2024-11-17T12:48:20,479 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-17T12:48:20,480 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-17T12:48:20,481 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T12:48:20,481 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-17T12:48:20,481 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-17T12:48:20,481 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer
2024-11-17T12:48:20,800 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:20,804 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:21,801 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:21,805 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:22,802 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:22,806 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:23,802 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:23,806 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:24,457 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:24,458 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:24,458 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:24,458 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:24,458 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:24,458 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:24,475 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:24,476 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:24,476 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:24,476 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:24,476 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:24,477 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:24,481 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:24,481 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:24,481 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:24,484 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:24,803 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:24,807 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:24,990 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-17T12:48:24,991 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:24,991 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:24,991 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:24,992 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:24,992 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:24,992 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:25,019 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:25,020 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:25,020 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:25,021 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:25,021 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:25,021 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:25,028 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:25,028 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:25,029 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:25,033 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:48:25,041 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-17T12:48:25,041 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling'
2024-11-17T12:48:25,804 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:25,807 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:26,804 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:26,808 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:27,805 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:27,808 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:28,806 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:28,809 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:29,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39777 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-17T12:48:29,729 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-17T12:48:29,730 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100
2024-11-17T12:48:29,733 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-17T12:48:29,733 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.
2024-11-17T12:48:29,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71., hostname=0d751a622496,34965,1731847698308, seqNum=2]
2024-11-17T12:48:29,743 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39777 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-17T12:48:29,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39777 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-17T12:48:29,750 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-17T12:48:29,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39777 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-17T12:48:29,752 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-17T12:48:29,753 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-17T12:48:29,806 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:29,809 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:29,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34965 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8
2024-11-17T12:48:29,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.
2024-11-17T12:48:29,919 INFO  [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 883c19deec29e3af1819980d5d6bbf71 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-17T12:48:29,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/.tmp/info/c8a8fc30542c412ebc66c7a77ba5a6be is 1080, key is row0001/info:/1731847709737/Put/seqid=0
2024-11-17T12:48:29,945 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741837_1013 (size=6033)
2024-11-17T12:48:29,945 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741837_1013 (size=6033)
2024-11-17T12:48:29,946 INFO  [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/.tmp/info/c8a8fc30542c412ebc66c7a77ba5a6be
2024-11-17T12:48:29,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/.tmp/info/c8a8fc30542c412ebc66c7a77ba5a6be as hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/info/c8a8fc30542c412ebc66c7a77ba5a6be
2024-11-17T12:48:29,959 INFO  [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/info/c8a8fc30542c412ebc66c7a77ba5a6be, entries=1, sequenceid=5, filesize=5.9 K
2024-11-17T12:48:29,960 INFO  [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 883c19deec29e3af1819980d5d6bbf71 in 41ms, sequenceid=5, compaction requested=false
2024-11-17T12:48:29,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 883c19deec29e3af1819980d5d6bbf71:

2024-11-17T12:48:29,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.
2024-11-17T12:48:29,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8
2024-11-17T12:48:29,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39777 {}] master.HMaster(4169): Remote procedure done, pid=8
2024-11-17T12:48:29,967 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-17T12:48:29,967 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 211 msec
2024-11-17T12:48:29,970 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 223 msec
2024-11-17T12:48:30,807 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:30,810 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:31,808 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:31,810 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:32,808 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:32,811 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:33,809 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:33,812 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:34,810 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:34,812 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:35,811 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:35,813 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:36,812 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:36,814 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:37,813 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:37,814 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:38,813 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:38,815 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:39,814 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:39,814 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224 after 68053ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor202.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:48:39,815 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:39,816 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta after 68043ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor202.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-17T12:48:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39777 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-17T12:48:39,840 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-17T12:48:39,843 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39777 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-17T12:48:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39777 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-17T12:48:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39777 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-17T12:48:39,846 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-17T12:48:39,847 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-17T12:48:39,847 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-17T12:48:40,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34965 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10
2024-11-17T12:48:40,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.
2024-11-17T12:48:40,001 INFO  [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 883c19deec29e3af1819980d5d6bbf71 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-17T12:48:40,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/.tmp/info/6002724b293349eba7db0e768b1efdae is 1080, key is row0002/info:/1731847719841/Put/seqid=0
2024-11-17T12:48:40,011 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741838_1014 (size=6033)
2024-11-17T12:48:40,011 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741838_1014 (size=6033)
2024-11-17T12:48:40,012 INFO  [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/.tmp/info/6002724b293349eba7db0e768b1efdae
2024-11-17T12:48:40,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/.tmp/info/6002724b293349eba7db0e768b1efdae as hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/info/6002724b293349eba7db0e768b1efdae
2024-11-17T12:48:40,024 INFO  [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/info/6002724b293349eba7db0e768b1efdae, entries=1, sequenceid=9, filesize=5.9 K
2024-11-17T12:48:40,025 INFO  [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 883c19deec29e3af1819980d5d6bbf71 in 24ms, sequenceid=9, compaction requested=false
2024-11-17T12:48:40,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 883c19deec29e3af1819980d5d6bbf71:

2024-11-17T12:48:40,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.
2024-11-17T12:48:40,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10
2024-11-17T12:48:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39777 {}] master.HMaster(4169): Remote procedure done, pid=10
2024-11-17T12:48:40,030 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-11-17T12:48:40,030 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 180 msec
2024-11-17T12:48:40,032 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 188 msec
2024-11-17T12:48:40,815 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:40,816 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:41,815 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:41,817 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:42,816 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:42,817 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:43,817 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:43,818 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:44,817 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:44,818 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:45,818 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:45,819 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:46,819 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:46,819 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:47,820 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:47,820 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:48,124 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-17T12:48:48,820 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:48,820 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:49,821 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:49,821 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:49,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39777 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-17T12:48:49,950 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-17T12:48:49,954 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C34965%2C1731847698308.1731847729954
2024-11-17T12:48:49,961 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:49,961 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:49,961 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:49,961 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:49,961 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:48:49,961 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/WALs/0d751a622496,34965,1731847698308/0d751a622496%2C34965%2C1731847698308.1731847698937 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/WALs/0d751a622496,34965,1731847698308/0d751a622496%2C34965%2C1731847698308.1731847729954
2024-11-17T12:48:49,963 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46531:46531),(127.0.0.1/127.0.0.1:45821:45821)]
2024-11-17T12:48:49,963 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/WALs/0d751a622496,34965,1731847698308/0d751a622496%2C34965%2C1731847698308.1731847698937 is not closed yet, will try archiving it next time
2024-11-17T12:48:49,964 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741833_1009 (size=5546)
2024-11-17T12:48:49,964 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39777 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-17T12:48:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39777 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-17T12:48:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39777 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-17T12:48:49,966 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-17T12:48:49,967 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-17T12:48:49,967 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-17T12:48:49,969 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741833_1009 (size=5546)
2024-11-17T12:48:50,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34965 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12
2024-11-17T12:48:50,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.
2024-11-17T12:48:50,121 INFO  [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 883c19deec29e3af1819980d5d6bbf71 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-17T12:48:50,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/.tmp/info/3238c9111bdf40c981d7faf6531eb65e is 1080, key is row0003/info:/1731847729952/Put/seqid=0
2024-11-17T12:48:50,140 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741840_1016 (size=6033)
2024-11-17T12:48:50,140 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741840_1016 (size=6033)
2024-11-17T12:48:50,141 INFO  [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/.tmp/info/3238c9111bdf40c981d7faf6531eb65e
2024-11-17T12:48:50,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/.tmp/info/3238c9111bdf40c981d7faf6531eb65e as hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/info/3238c9111bdf40c981d7faf6531eb65e
2024-11-17T12:48:50,153 INFO  [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/info/3238c9111bdf40c981d7faf6531eb65e, entries=1, sequenceid=13, filesize=5.9 K
2024-11-17T12:48:50,155 INFO  [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 883c19deec29e3af1819980d5d6bbf71 in 33ms, sequenceid=13, compaction requested=true
2024-11-17T12:48:50,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 883c19deec29e3af1819980d5d6bbf71:

2024-11-17T12:48:50,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.
2024-11-17T12:48:50,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12
2024-11-17T12:48:50,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39777 {}] master.HMaster(4169): Remote procedure done, pid=12
2024-11-17T12:48:50,159 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-11-17T12:48:50,159 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 190 msec
2024-11-17T12:48:50,161 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 196 msec
2024-11-17T12:48:50,822 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:50,822 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:51,823 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:51,823 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:52,824 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:52,824 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:53,824 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:53,824 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:54,825 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:54,825 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:55,826 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:55,826 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:56,827 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:56,827 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:57,828 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:57,828 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:58,829 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:58,829 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:59,829 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:59,829 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:48:59,937 INFO  [master/0d751a622496:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-17T12:48:59,937 INFO  [master/0d751a622496:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-17T12:49:00,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39777 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-17T12:49:00,060 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-17T12:49:00,060 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T12:49:00,062 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-17T12:49:00,062 DEBUG [Time-limited test {}] regionserver.HStore(1541): 883c19deec29e3af1819980d5d6bbf71/info is initiating minor compaction (all files)
2024-11-17T12:49:00,062 INFO  [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-17T12:49:00,062 INFO  [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:00,062 INFO  [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 883c19deec29e3af1819980d5d6bbf71/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.
2024-11-17T12:49:00,063 INFO  [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/info/c8a8fc30542c412ebc66c7a77ba5a6be, hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/info/6002724b293349eba7db0e768b1efdae, hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/info/3238c9111bdf40c981d7faf6531eb65e] into tmpdir=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/.tmp, totalSize=17.7 K
2024-11-17T12:49:00,063 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c8a8fc30542c412ebc66c7a77ba5a6be, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731847709737
2024-11-17T12:49:00,064 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 6002724b293349eba7db0e768b1efdae, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1731847719841
2024-11-17T12:49:00,064 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 3238c9111bdf40c981d7faf6531eb65e, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731847729952
2024-11-17T12:49:00,078 INFO  [Time-limited test {}] throttle.PressureAwareThroughputController(145): 883c19deec29e3af1819980d5d6bbf71#info#compaction#45 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T12:49:00,079 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/.tmp/info/021687b9656a4fe3a013ec74973b5f13 is 1080, key is row0001/info:/1731847709737/Put/seqid=0
2024-11-17T12:49:00,090 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741841_1017 (size=8296)
2024-11-17T12:49:00,091 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741841_1017 (size=8296)
2024-11-17T12:49:00,099 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/.tmp/info/021687b9656a4fe3a013ec74973b5f13 as hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/info/021687b9656a4fe3a013ec74973b5f13
2024-11-17T12:49:00,107 INFO  [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 883c19deec29e3af1819980d5d6bbf71/info of 883c19deec29e3af1819980d5d6bbf71 into 021687b9656a4fe3a013ec74973b5f13(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T12:49:00,107 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 883c19deec29e3af1819980d5d6bbf71:

2024-11-17T12:49:00,110 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C34965%2C1731847698308.1731847740109
2024-11-17T12:49:00,116 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:00,116 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:00,116 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:00,116 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:00,116 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:00,116 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/WALs/0d751a622496,34965,1731847698308/0d751a622496%2C34965%2C1731847698308.1731847729954 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/WALs/0d751a622496,34965,1731847698308/0d751a622496%2C34965%2C1731847698308.1731847740109
2024-11-17T12:49:00,117 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46531:46531),(127.0.0.1/127.0.0.1:45821:45821)]
2024-11-17T12:49:00,117 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/WALs/0d751a622496,34965,1731847698308/0d751a622496%2C34965%2C1731847698308.1731847729954 is not closed yet, will try archiving it next time
2024-11-17T12:49:00,118 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/WALs/0d751a622496,34965,1731847698308/0d751a622496%2C34965%2C1731847698308.1731847698937 to hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/oldWALs/0d751a622496%2C34965%2C1731847698308.1731847698937
2024-11-17T12:49:00,119 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741839_1015 (size=2520)
2024-11-17T12:49:00,119 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741839_1015 (size=2520)
2024-11-17T12:49:00,119 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39777 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-17T12:49:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39777 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-17T12:49:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39777 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-17T12:49:00,122 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-17T12:49:00,123 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-17T12:49:00,123 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-17T12:49:00,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34965 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14
2024-11-17T12:49:00,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.
2024-11-17T12:49:00,276 INFO  [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 883c19deec29e3af1819980d5d6bbf71 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-17T12:49:00,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/.tmp/info/beb19c0c363e48c687e81f6cd990b00d is 1080, key is row0000/info:/1731847740108/Put/seqid=0
2024-11-17T12:49:00,308 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741843_1019 (size=6033)
2024-11-17T12:49:00,308 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741843_1019 (size=6033)
2024-11-17T12:49:00,309 INFO  [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/.tmp/info/beb19c0c363e48c687e81f6cd990b00d
2024-11-17T12:49:00,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/.tmp/info/beb19c0c363e48c687e81f6cd990b00d as hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/info/beb19c0c363e48c687e81f6cd990b00d
2024-11-17T12:49:00,324 INFO  [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/info/beb19c0c363e48c687e81f6cd990b00d, entries=1, sequenceid=18, filesize=5.9 K
2024-11-17T12:49:00,325 INFO  [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 883c19deec29e3af1819980d5d6bbf71 in 49ms, sequenceid=18, compaction requested=false
2024-11-17T12:49:00,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 883c19deec29e3af1819980d5d6bbf71:

2024-11-17T12:49:00,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.
2024-11-17T12:49:00,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14
2024-11-17T12:49:00,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39777 {}] master.HMaster(4169): Remote procedure done, pid=14
2024-11-17T12:49:00,329 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-11-17T12:49:00,329 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 204 msec
2024-11-17T12:49:00,332 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 211 msec
2024-11-17T12:49:00,830 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:00,830 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:01,831 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:01,831 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:02,831 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:02,831 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:03,832 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:03,832 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:04,833 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:04,833 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:04,997 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 883c19deec29e3af1819980d5d6bbf71, had cached 0 bytes from a total of 14329
2024-11-17T12:49:05,834 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:05,834 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:06,834 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:06,834 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:07,835 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:07,835 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:08,836 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:08,836 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:09,836 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:09,836 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39777 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-17T12:49:10,199 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-17T12:49:10,202 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C34965%2C1731847698308.1731847750202
2024-11-17T12:49:10,207 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:10,207 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:10,207 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:10,207 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:10,208 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:10,208 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/WALs/0d751a622496,34965,1731847698308/0d751a622496%2C34965%2C1731847698308.1731847740109 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/WALs/0d751a622496,34965,1731847698308/0d751a622496%2C34965%2C1731847698308.1731847750202
2024-11-17T12:49:10,208 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46531:46531),(127.0.0.1/127.0.0.1:45821:45821)]
2024-11-17T12:49:10,209 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/WALs/0d751a622496,34965,1731847698308/0d751a622496%2C34965%2C1731847698308.1731847740109 is not closed yet, will try archiving it next time
2024-11-17T12:49:10,209 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/WALs/0d751a622496,34965,1731847698308/0d751a622496%2C34965%2C1731847698308.1731847729954 to hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/oldWALs/0d751a622496%2C34965%2C1731847698308.1731847729954
2024-11-17T12:49:10,209 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-17T12:49:10,209 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T12:49:10,209 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T12:49:10,209 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:49:10,209 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:49:10,209 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-17T12:49:10,209 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741842_1018 (size=2026)
2024-11-17T12:49:10,209 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-17T12:49:10,209 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1916862285, stopped=false
2024-11-17T12:49:10,209 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741842_1018 (size=2026)
2024-11-17T12:49:10,209 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0d751a622496,39777,1731847698143
2024-11-17T12:49:10,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T12:49:10,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34965-0x1014a2049940001, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T12:49:10,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:10,299 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T12:49:10,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34965-0x1014a2049940001, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:10,299 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T12:49:10,299 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T12:49:10,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:49:10,299 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:49:10,299 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0d751a622496,34965,1731847698308' *****
2024-11-17T12:49:10,299 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-17T12:49:10,300 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34965-0x1014a2049940001, quorum=127.0.0.1:56716, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:49:10,300 INFO  [RS:0;0d751a622496:34965 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-17T12:49:10,300 INFO  [RS:0;0d751a622496:34965 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-17T12:49:10,300 INFO  [RS:0;0d751a622496:34965 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-17T12:49:10,300 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-17T12:49:10,300 INFO  [RS:0;0d751a622496:34965 {}] regionserver.HRegionServer(3091): Received CLOSE for 883c19deec29e3af1819980d5d6bbf71
2024-11-17T12:49:10,301 INFO  [RS:0;0d751a622496:34965 {}] regionserver.HRegionServer(959): stopping server 0d751a622496,34965,1731847698308
2024-11-17T12:49:10,301 INFO  [RS:0;0d751a622496:34965 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T12:49:10,301 INFO  [RS:0;0d751a622496:34965 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0d751a622496:34965.
2024-11-17T12:49:10,301 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 883c19deec29e3af1819980d5d6bbf71, disabling compactions & flushes
2024-11-17T12:49:10,301 DEBUG [RS:0;0d751a622496:34965 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T12:49:10,301 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.
2024-11-17T12:49:10,301 DEBUG [RS:0;0d751a622496:34965 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:49:10,301 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.
2024-11-17T12:49:10,301 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71. after waiting 0 ms
2024-11-17T12:49:10,301 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.
2024-11-17T12:49:10,301 INFO  [RS:0;0d751a622496:34965 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-17T12:49:10,301 INFO  [RS:0;0d751a622496:34965 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-17T12:49:10,301 INFO  [RS:0;0d751a622496:34965 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-17T12:49:10,301 INFO  [RS:0;0d751a622496:34965 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-17T12:49:10,301 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 883c19deec29e3af1819980d5d6bbf71 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-17T12:49:10,301 INFO  [RS:0;0d751a622496:34965 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-17T12:49:10,301 DEBUG [RS:0;0d751a622496:34965 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 883c19deec29e3af1819980d5d6bbf71=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.}
2024-11-17T12:49:10,302 DEBUG [RS:0;0d751a622496:34965 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 883c19deec29e3af1819980d5d6bbf71
2024-11-17T12:49:10,302 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T12:49:10,302 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T12:49:10,302 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T12:49:10,302 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T12:49:10,302 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T12:49:10,302 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB
2024-11-17T12:49:10,306 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/.tmp/info/aed3dd9007ba4bfd8a72147c1bad79af is 1080, key is row0001/info:/1731847750200/Put/seqid=0
2024-11-17T12:49:10,311 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741845_1021 (size=6033)
2024-11-17T12:49:10,311 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741845_1021 (size=6033)
2024-11-17T12:49:10,311 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/.tmp/info/aed3dd9007ba4bfd8a72147c1bad79af
2024-11-17T12:49:10,318 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/.tmp/info/aed3dd9007ba4bfd8a72147c1bad79af as hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/info/aed3dd9007ba4bfd8a72147c1bad79af
2024-11-17T12:49:10,320 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/hbase/meta/1588230740/.tmp/info/44516012727749d495a3fb4fd6174b9a is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71./info:regioninfo/1731847700018/Put/seqid=0
2024-11-17T12:49:10,323 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/info/aed3dd9007ba4bfd8a72147c1bad79af, entries=1, sequenceid=22, filesize=5.9 K
2024-11-17T12:49:10,324 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 883c19deec29e3af1819980d5d6bbf71 in 23ms, sequenceid=22, compaction requested=true
2024-11-17T12:49:10,327 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/info/c8a8fc30542c412ebc66c7a77ba5a6be, hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/info/6002724b293349eba7db0e768b1efdae, hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/info/3238c9111bdf40c981d7faf6531eb65e] to archive
2024-11-17T12:49:10,328 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-17T12:49:10,328 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741846_1022 (size=7308)
2024-11-17T12:49:10,329 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741846_1022 (size=7308)
2024-11-17T12:49:10,329 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/hbase/meta/1588230740/.tmp/info/44516012727749d495a3fb4fd6174b9a
2024-11-17T12:49:10,330 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/info/c8a8fc30542c412ebc66c7a77ba5a6be to hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/info/c8a8fc30542c412ebc66c7a77ba5a6be
2024-11-17T12:49:10,332 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/info/6002724b293349eba7db0e768b1efdae to hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/info/6002724b293349eba7db0e768b1efdae
2024-11-17T12:49:10,333 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/info/3238c9111bdf40c981d7faf6531eb65e to hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/info/3238c9111bdf40c981d7faf6531eb65e
2024-11-17T12:49:10,333 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=0d751a622496:39777 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-11-17T12:49:10,334 WARN  [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [c8a8fc30542c412ebc66c7a77ba5a6be=6033, 6002724b293349eba7db0e768b1efdae=6033, 3238c9111bdf40c981d7faf6531eb65e=6033]
2024-11-17T12:49:10,337 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/883c19deec29e3af1819980d5d6bbf71/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1
2024-11-17T12:49:10,338 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.
2024-11-17T12:49:10,338 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 883c19deec29e3af1819980d5d6bbf71:
Waiting for close lock at 1731847750301Running coprocessor pre-close hooks at 1731847750301Disabling compacts and flushes for region at 1731847750301Disabling writes for close at 1731847750301Obtaining lock to block concurrent updates at 1731847750301Preparing flush snapshotting stores in 883c19deec29e3af1819980d5d6bbf71 at 1731847750301Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1731847750302 (+1 ms)Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71. at 1731847750302Flushing 883c19deec29e3af1819980d5d6bbf71/info: creating writer at 1731847750302Flushing 883c19deec29e3af1819980d5d6bbf71/info: appending metadata at 1731847750305 (+3 ms)Flushing 883c19deec29e3af1819980d5d6bbf71/info: closing flushed file at 1731847750305Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24d0a4f7: reopening flushed file at 1731847750317 (+12 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 883c19deec29e3af1819980d5d6bbf71 in 23ms, sequenceid=22, compaction requested=true at 1731847750324 (+7 ms)Writing region close event to WAL at 1731847750334 (+10 ms)Running coprocessor post-close hooks at 1731847750338 (+4 ms)Closed at 1731847750338
2024-11-17T12:49:10,338 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731847699651.883c19deec29e3af1819980d5d6bbf71.
2024-11-17T12:49:10,350 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/hbase/meta/1588230740/.tmp/ns/324036dcf8e94ed697c17689672a32a6 is 43, key is default/ns:d/1731847699525/Put/seqid=0
2024-11-17T12:49:10,354 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741847_1023 (size=5153)
2024-11-17T12:49:10,354 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741847_1023 (size=5153)
2024-11-17T12:49:10,355 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/hbase/meta/1588230740/.tmp/ns/324036dcf8e94ed697c17689672a32a6
2024-11-17T12:49:10,376 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/hbase/meta/1588230740/.tmp/table/02b1309f408a483b8c8ce89379773379 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1731847700029/Put/seqid=0
2024-11-17T12:49:10,381 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741848_1024 (size=5508)
2024-11-17T12:49:10,381 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741848_1024 (size=5508)
2024-11-17T12:49:10,382 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/hbase/meta/1588230740/.tmp/table/02b1309f408a483b8c8ce89379773379
2024-11-17T12:49:10,387 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/hbase/meta/1588230740/.tmp/info/44516012727749d495a3fb4fd6174b9a as hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/hbase/meta/1588230740/info/44516012727749d495a3fb4fd6174b9a
2024-11-17T12:49:10,393 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/hbase/meta/1588230740/info/44516012727749d495a3fb4fd6174b9a, entries=10, sequenceid=11, filesize=7.1 K
2024-11-17T12:49:10,394 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/hbase/meta/1588230740/.tmp/ns/324036dcf8e94ed697c17689672a32a6 as hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/hbase/meta/1588230740/ns/324036dcf8e94ed697c17689672a32a6
2024-11-17T12:49:10,400 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/hbase/meta/1588230740/ns/324036dcf8e94ed697c17689672a32a6, entries=2, sequenceid=11, filesize=5.0 K
2024-11-17T12:49:10,401 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/hbase/meta/1588230740/.tmp/table/02b1309f408a483b8c8ce89379773379 as hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/hbase/meta/1588230740/table/02b1309f408a483b8c8ce89379773379
2024-11-17T12:49:10,407 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/hbase/meta/1588230740/table/02b1309f408a483b8c8ce89379773379, entries=2, sequenceid=11, filesize=5.4 K
2024-11-17T12:49:10,409 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 106ms, sequenceid=11, compaction requested=false
2024-11-17T12:49:10,413 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-17T12:49:10,414 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T12:49:10,414 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T12:49:10,414 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731847750301Running coprocessor pre-close hooks at 1731847750301Disabling compacts and flushes for region at 1731847750301Disabling writes for close at 1731847750302 (+1 ms)Obtaining lock to block concurrent updates at 1731847750302Preparing flush snapshotting stores in 1588230740 at 1731847750302Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1731847750302Flushing stores of hbase:meta,,1.1588230740 at 1731847750303 (+1 ms)Flushing 1588230740/info: creating writer at 1731847750303Flushing 1588230740/info: appending metadata at 1731847750320 (+17 ms)Flushing 1588230740/info: closing flushed file at 1731847750320Flushing 1588230740/ns: creating writer at 1731847750334 (+14 ms)Flushing 1588230740/ns: appending metadata at 1731847750350 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731847750350Flushing 1588230740/table: creating writer at 1731847750360 (+10 ms)Flushing 1588230740/table: appending metadata at 1731847750375 (+15 ms)Flushing 1588230740/table: closing flushed file at 1731847750375Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bbf3c7f: reopening flushed file at 1731847750387 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f5cafa1: reopening flushed file at 1731847750393 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@390621ce: reopening flushed file at 1731847750400 (+7 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 106ms, sequenceid=11, compaction requested=false at 1731847750409 (+9 ms)Writing region close event to WAL at 1731847750410 (+1 ms)Running coprocessor post-close hooks at 1731847750413 (+3 ms)Closed at 1731847750414 (+1 ms)
2024-11-17T12:49:10,414 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-17T12:49:10,479 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T12:49:10,480 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-17T12:49:10,480 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-17T12:49:10,502 INFO  [RS:0;0d751a622496:34965 {}] regionserver.HRegionServer(976): stopping server 0d751a622496,34965,1731847698308; all regions closed.
2024-11-17T12:49:10,502 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:10,502 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:10,503 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:10,503 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:10,503 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:10,505 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741834_1010 (size=3306)
2024-11-17T12:49:10,505 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741834_1010 (size=3306)
2024-11-17T12:49:10,508 DEBUG [RS:0;0d751a622496:34965 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/oldWALs
2024-11-17T12:49:10,508 INFO  [RS:0;0d751a622496:34965 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0d751a622496%2C34965%2C1731847698308.meta:.meta(num 1731847699449)
2024-11-17T12:49:10,508 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:10,508 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:10,508 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:10,508 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:10,508 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:10,510 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741844_1020 (size=1252)
2024-11-17T12:49:10,510 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741844_1020 (size=1252)
2024-11-17T12:49:10,514 DEBUG [RS:0;0d751a622496:34965 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/oldWALs
2024-11-17T12:49:10,514 INFO  [RS:0;0d751a622496:34965 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0d751a622496%2C34965%2C1731847698308:(num 1731847750202)
2024-11-17T12:49:10,514 DEBUG [RS:0;0d751a622496:34965 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:49:10,514 INFO  [RS:0;0d751a622496:34965 {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T12:49:10,514 INFO  [RS:0;0d751a622496:34965 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T12:49:10,515 INFO  [RS:0;0d751a622496:34965 {}] hbase.ChoreService(370): Chore service for: regionserver/0d751a622496:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-17T12:49:10,515 INFO  [RS:0;0d751a622496:34965 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T12:49:10,515 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T12:49:10,515 INFO  [RS:0;0d751a622496:34965 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34965
2024-11-17T12:49:10,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34965-0x1014a2049940001, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0d751a622496,34965,1731847698308
2024-11-17T12:49:10,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T12:49:10,547 INFO  [RS:0;0d751a622496:34965 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T12:49:10,548 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher.
java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$363/0x00007f52e0903a08@35cff532 rejected from java.util.concurrent.ThreadPoolExecutor@70cd6def[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 14]
	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?]
	at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?]
	at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4]
	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4]
2024-11-17T12:49:10,557 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0d751a622496,34965,1731847698308]
2024-11-17T12:49:10,565 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0d751a622496,34965,1731847698308 already deleted, retry=false
2024-11-17T12:49:10,565 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0d751a622496,34965,1731847698308 expired; onlineServers=0
2024-11-17T12:49:10,565 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0d751a622496,39777,1731847698143' *****
2024-11-17T12:49:10,565 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-17T12:49:10,565 INFO  [M:0;0d751a622496:39777 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T12:49:10,565 INFO  [M:0;0d751a622496:39777 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T12:49:10,565 DEBUG [M:0;0d751a622496:39777 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-17T12:49:10,565 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-17T12:49:10,565 DEBUG [M:0;0d751a622496:39777 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-17T12:49:10,565 DEBUG [master/0d751a622496:0:becomeActiveMaster-HFileCleaner.large.0-1731847698652 {}] cleaner.HFileCleaner(306): Exit Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.large.0-1731847698652,5,FailOnTimeoutGroup]
2024-11-17T12:49:10,565 DEBUG [master/0d751a622496:0:becomeActiveMaster-HFileCleaner.small.0-1731847698652 {}] cleaner.HFileCleaner(306): Exit Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.small.0-1731847698652,5,FailOnTimeoutGroup]
2024-11-17T12:49:10,566 INFO  [M:0;0d751a622496:39777 {}] hbase.ChoreService(370): Chore service for: master/0d751a622496:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-17T12:49:10,566 INFO  [M:0;0d751a622496:39777 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T12:49:10,566 DEBUG [M:0;0d751a622496:39777 {}] master.HMaster(1795): Stopping service threads
2024-11-17T12:49:10,566 INFO  [M:0;0d751a622496:39777 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-17T12:49:10,566 INFO  [M:0;0d751a622496:39777 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T12:49:10,566 INFO  [M:0;0d751a622496:39777 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-17T12:49:10,566 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-17T12:49:10,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-17T12:49:10,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:10,581 DEBUG [M:0;0d751a622496:39777 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false
2024-11-17T12:49:10,582 DEBUG [M:0;0d751a622496:39777 {}] master.ActiveMasterManager(353): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master
2024-11-17T12:49:10,582 INFO  [M:0;0d751a622496:39777 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/.lastflushedseqids
2024-11-17T12:49:10,589 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741849_1025 (size=130)
2024-11-17T12:49:10,589 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741849_1025 (size=130)
2024-11-17T12:49:10,590 INFO  [M:0;0d751a622496:39777 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-17T12:49:10,590 INFO  [M:0;0d751a622496:39777 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-17T12:49:10,590 DEBUG [M:0;0d751a622496:39777 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T12:49:10,590 INFO  [M:0;0d751a622496:39777 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:49:10,590 DEBUG [M:0;0d751a622496:39777 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:49:10,591 DEBUG [M:0;0d751a622496:39777 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T12:49:10,591 DEBUG [M:0;0d751a622496:39777 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:49:10,591 INFO  [M:0;0d751a622496:39777 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.60 KB heapSize=55.01 KB
2024-11-17T12:49:10,608 DEBUG [M:0;0d751a622496:39777 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c51ff1e5ba934cd2baddc6deebb7415a is 82, key is hbase:meta,,1/info:regioninfo/1731847699475/Put/seqid=0
2024-11-17T12:49:10,612 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741850_1026 (size=5672)
2024-11-17T12:49:10,613 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741850_1026 (size=5672)
2024-11-17T12:49:10,613 INFO  [M:0;0d751a622496:39777 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c51ff1e5ba934cd2baddc6deebb7415a
2024-11-17T12:49:10,638 DEBUG [M:0;0d751a622496:39777 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/095396944a5a449ea3b6d341d655a95d is 798, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731847700035/Put/seqid=0
2024-11-17T12:49:10,643 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741851_1027 (size=7824)
2024-11-17T12:49:10,644 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741851_1027 (size=7824)
2024-11-17T12:49:10,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34965-0x1014a2049940001, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:49:10,657 INFO  [RS:0;0d751a622496:34965 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T12:49:10,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34965-0x1014a2049940001, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:49:10,657 INFO  [RS:0;0d751a622496:34965 {}] regionserver.HRegionServer(1031): Exiting; stopping=0d751a622496,34965,1731847698308; zookeeper connection closed.
2024-11-17T12:49:10,657 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3f3062e7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3f3062e7
2024-11-17T12:49:10,657 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-17T12:49:10,804 INFO  [regionserver/0d751a622496:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T12:49:10,837 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:10,837 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:11,044 INFO  [M:0;0d751a622496:39777 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.00 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/095396944a5a449ea3b6d341d655a95d
2024-11-17T12:49:11,052 INFO  [M:0;0d751a622496:39777 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 095396944a5a449ea3b6d341d655a95d
2024-11-17T12:49:11,070 DEBUG [M:0;0d751a622496:39777 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9ff48c40619c4da8870dc7238c81e2cb is 69, key is 0d751a622496,34965,1731847698308/rs:state/1731847698782/Put/seqid=0
2024-11-17T12:49:11,075 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741852_1028 (size=5156)
2024-11-17T12:49:11,075 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741852_1028 (size=5156)
2024-11-17T12:49:11,076 INFO  [M:0;0d751a622496:39777 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9ff48c40619c4da8870dc7238c81e2cb
2024-11-17T12:49:11,095 DEBUG [M:0;0d751a622496:39777 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9644672ae9e74a8ba1d46622f02c7a45 is 52, key is load_balancer_on/state:d/1731847699647/Put/seqid=0
2024-11-17T12:49:11,099 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741853_1029 (size=5056)
2024-11-17T12:49:11,099 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741853_1029 (size=5056)
2024-11-17T12:49:11,100 INFO  [M:0;0d751a622496:39777 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9644672ae9e74a8ba1d46622f02c7a45
2024-11-17T12:49:11,106 DEBUG [M:0;0d751a622496:39777 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c51ff1e5ba934cd2baddc6deebb7415a as hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c51ff1e5ba934cd2baddc6deebb7415a
2024-11-17T12:49:11,111 INFO  [M:0;0d751a622496:39777 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c51ff1e5ba934cd2baddc6deebb7415a, entries=8, sequenceid=121, filesize=5.5 K
2024-11-17T12:49:11,112 DEBUG [M:0;0d751a622496:39777 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/095396944a5a449ea3b6d341d655a95d as hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/095396944a5a449ea3b6d341d655a95d
2024-11-17T12:49:11,117 INFO  [M:0;0d751a622496:39777 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 095396944a5a449ea3b6d341d655a95d
2024-11-17T12:49:11,117 INFO  [M:0;0d751a622496:39777 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/095396944a5a449ea3b6d341d655a95d, entries=14, sequenceid=121, filesize=7.6 K
2024-11-17T12:49:11,118 DEBUG [M:0;0d751a622496:39777 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9ff48c40619c4da8870dc7238c81e2cb as hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9ff48c40619c4da8870dc7238c81e2cb
2024-11-17T12:49:11,123 INFO  [M:0;0d751a622496:39777 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9ff48c40619c4da8870dc7238c81e2cb, entries=1, sequenceid=121, filesize=5.0 K
2024-11-17T12:49:11,124 DEBUG [M:0;0d751a622496:39777 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9644672ae9e74a8ba1d46622f02c7a45 as hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/9644672ae9e74a8ba1d46622f02c7a45
2024-11-17T12:49:11,129 INFO  [M:0;0d751a622496:39777 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40479/user/jenkins/test-data/7b9bcc88-8b42-5851-5edc-ab3c2a24e226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/9644672ae9e74a8ba1d46622f02c7a45, entries=1, sequenceid=121, filesize=4.9 K
2024-11-17T12:49:11,130 INFO  [M:0;0d751a622496:39777 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.60 KB/44647, heapSize ~54.95 KB/56264, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 539ms, sequenceid=121, compaction requested=false
2024-11-17T12:49:11,132 INFO  [M:0;0d751a622496:39777 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:49:11,132 DEBUG [M:0;0d751a622496:39777 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731847750590Disabling compacts and flushes for region at 1731847750590Disabling writes for close at 1731847750591 (+1 ms)Obtaining lock to block concurrent updates at 1731847750591Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731847750591Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44647, getHeapSize=56264, getOffHeapSize=0, getCellsCount=140 at 1731847750591Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731847750592 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731847750592Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731847750607 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731847750607Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731847750618 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731847750637 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731847750637Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731847751052 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731847751070 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731847751070Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731847751080 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731847751094 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731847751095 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38712b1d: reopening flushed file at 1731847751105 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ba79124: reopening flushed file at 1731847751111 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28b007e8: reopening flushed file at 1731847751117 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54755a25: reopening flushed file at 1731847751123 (+6 ms)Finished flush of dataSize ~43.60 KB/44647, heapSize ~54.95 KB/56264, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 539ms, sequenceid=121, compaction requested=false at 1731847751130 (+7 ms)Writing region close event to WAL at 1731847751132 (+2 ms)Closed at 1731847751132
2024-11-17T12:49:11,132 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:11,132 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:11,132 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:11,132 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:11,132 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:11,134 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32911 is added to blk_1073741830_1006 (size=53044)
2024-11-17T12:49:11,134 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38897 is added to blk_1073741830_1006 (size=53044)
2024-11-17T12:49:11,135 INFO  [M:0;0d751a622496:39777 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-17T12:49:11,135 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T12:49:11,135 INFO  [M:0;0d751a622496:39777 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39777
2024-11-17T12:49:11,135 INFO  [M:0;0d751a622496:39777 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T12:49:11,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:49:11,289 INFO  [M:0;0d751a622496:39777 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T12:49:11,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39777-0x1014a2049940000, quorum=127.0.0.1:56716, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:49:11,328 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5cdf1f1c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:49:11,328 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@51e1410e{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:49:11,328 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:49:11,329 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41bc1801{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:49:11,329 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5894e22d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/hadoop.log.dir/,STOPPED}
2024-11-17T12:49:11,331 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T12:49:11,331 WARN  [BP-1721540678-172.17.0.2-1731847696414 heartbeating to localhost/127.0.0.1:40479 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T12:49:11,331 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T12:49:11,331 WARN  [BP-1721540678-172.17.0.2-1731847696414 heartbeating to localhost/127.0.0.1:40479 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1721540678-172.17.0.2-1731847696414 (Datanode Uuid 0a8dfbd1-2528-452a-bcd4-4eaadcb0c577) service to localhost/127.0.0.1:40479
2024-11-17T12:49:11,332 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/cluster_389f1386-0ea4-b3cd-9518-5fd758751bda/data/data3/current/BP-1721540678-172.17.0.2-1731847696414 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:49:11,332 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/cluster_389f1386-0ea4-b3cd-9518-5fd758751bda/data/data4/current/BP-1721540678-172.17.0.2-1731847696414 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:49:11,332 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T12:49:11,334 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2526c219{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:49:11,335 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@74e6f5d9{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:49:11,335 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:49:11,335 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4bd1d692{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:49:11,335 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@10c2896a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/hadoop.log.dir/,STOPPED}
2024-11-17T12:49:11,336 WARN  [BP-1721540678-172.17.0.2-1731847696414 heartbeating to localhost/127.0.0.1:40479 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T12:49:11,336 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T12:49:11,337 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T12:49:11,337 WARN  [BP-1721540678-172.17.0.2-1731847696414 heartbeating to localhost/127.0.0.1:40479 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1721540678-172.17.0.2-1731847696414 (Datanode Uuid 1088402a-af37-4455-98d9-ff57b6e5ec20) service to localhost/127.0.0.1:40479
2024-11-17T12:49:11,337 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/cluster_389f1386-0ea4-b3cd-9518-5fd758751bda/data/data1/current/BP-1721540678-172.17.0.2-1731847696414 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:49:11,337 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/cluster_389f1386-0ea4-b3cd-9518-5fd758751bda/data/data2/current/BP-1721540678-172.17.0.2-1731847696414 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:49:11,338 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T12:49:11,344 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4d36967f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T12:49:11,345 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4d389c80{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:49:11,345 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:49:11,345 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7b8edabe{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:49:11,345 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1c41fb6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/hadoop.log.dir/,STOPPED}
2024-11-17T12:49:11,352 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-17T12:49:11,376 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-17T12:49:11,385 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=204 (was 179)
Potentially hanging thread: nioEventLoopGroup-37-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:40479
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40479
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40479 from jenkins.hfs.5
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-35-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40479
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40479 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-34-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40479
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:40479
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40479 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=483 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=220 (was 220), ProcessCount=11 (was 11), AvailableMemoryMB=5158 (was 5306)
2024-11-17T12:49:11,565 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=204, OpenFileDescriptor=483, MaxFileDescriptor=1048576, SystemLoadAverage=220, ProcessCount=11, AvailableMemoryMB=5159
2024-11-17T12:49:11,565 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-17T12:49:11,565 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/hadoop.log.dir so I do NOT create it in target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29
2024-11-17T12:49:11,565 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ecfd7d62-6f33-e1fa-634b-ef4b0c65c570/hadoop.tmp.dir so I do NOT create it in target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29
2024-11-17T12:49:11,565 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/cluster_c97ab916-9cf5-6219-161a-81127106f55c, deleteOnExit=true
2024-11-17T12:49:11,565 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-17T12:49:11,565 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/test.cache.data in system properties and HBase conf
2024-11-17T12:49:11,566 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/hadoop.tmp.dir in system properties and HBase conf
2024-11-17T12:49:11,566 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/hadoop.log.dir in system properties and HBase conf
2024-11-17T12:49:11,566 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-17T12:49:11,566 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-17T12:49:11,566 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-17T12:49:11,566 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-17T12:49:11,566 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-17T12:49:11,566 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-17T12:49:11,566 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-17T12:49:11,566 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T12:49:11,566 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-17T12:49:11,567 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-17T12:49:11,567 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T12:49:11,567 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T12:49:11,567 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-17T12:49:11,567 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/nfs.dump.dir in system properties and HBase conf
2024-11-17T12:49:11,567 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/java.io.tmpdir in system properties and HBase conf
2024-11-17T12:49:11,567 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T12:49:11,567 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-17T12:49:11,567 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-17T12:49:11,583 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T12:49:11,838 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:11,838 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:11,946 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:49:11,951 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:49:11,958 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:49:11,958 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:49:11,958 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-17T12:49:11,959 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:49:11,960 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@78512cf7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:49:11,960 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@47cbf00e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:49:12,067 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6a6db152{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/java.io.tmpdir/jetty-localhost-36827-hadoop-hdfs-3_4_1-tests_jar-_-any-2064409825154616014/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T12:49:12,067 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@79aa061{HTTP/1.1, (http/1.1)}{localhost:36827}
2024-11-17T12:49:12,067 INFO  [Time-limited test {}] server.Server(415): Started @247309ms
2024-11-17T12:49:12,080 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T12:49:12,355 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:49:12,358 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:49:12,359 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:49:12,359 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:49:12,359 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-17T12:49:12,360 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b4ed6ad{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:49:12,360 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ab86f9f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:49:12,466 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@444decb3{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/java.io.tmpdir/jetty-localhost-36365-hadoop-hdfs-3_4_1-tests_jar-_-any-6496075469265012733/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:49:12,466 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@216b0c63{HTTP/1.1, (http/1.1)}{localhost:36365}
2024-11-17T12:49:12,466 INFO  [Time-limited test {}] server.Server(415): Started @247708ms
2024-11-17T12:49:12,467 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T12:49:12,494 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:49:12,497 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:49:12,498 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:49:12,498 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:49:12,498 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-17T12:49:12,499 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5bbaae64{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:49:12,499 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7cbc9241{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:49:12,601 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4b81e396{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/java.io.tmpdir/jetty-localhost-42081-hadoop-hdfs-3_4_1-tests_jar-_-any-14909163240682321110/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:49:12,602 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2058e2af{HTTP/1.1, (http/1.1)}{localhost:42081}
2024-11-17T12:49:12,602 INFO  [Time-limited test {}] server.Server(415): Started @247844ms
2024-11-17T12:49:12,603 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T12:49:12,838 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:12,838 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:13,254 WARN  [Thread-1968 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/cluster_c97ab916-9cf5-6219-161a-81127106f55c/data/data2/current/BP-903435966-172.17.0.2-1731847751586/current, will proceed with Du for space computation calculation, 
2024-11-17T12:49:13,254 WARN  [Thread-1967 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/cluster_c97ab916-9cf5-6219-161a-81127106f55c/data/data1/current/BP-903435966-172.17.0.2-1731847751586/current, will proceed with Du for space computation calculation, 
2024-11-17T12:49:13,274 WARN  [Thread-1931 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T12:49:13,276 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x86e17bf191c349f7 with lease ID 0x7b0805c4811127: Processing first storage report for DS-22170c91-0b22-4235-b1bb-b90e956c18c8 from datanode DatanodeRegistration(127.0.0.1:39689, datanodeUuid=dd44247c-6fba-49b4-a979-dafe76384873, infoPort=35291, infoSecurePort=0, ipcPort=35789, storageInfo=lv=-57;cid=testClusterID;nsid=1718931293;c=1731847751586)
2024-11-17T12:49:13,276 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x86e17bf191c349f7 with lease ID 0x7b0805c4811127: from storage DS-22170c91-0b22-4235-b1bb-b90e956c18c8 node DatanodeRegistration(127.0.0.1:39689, datanodeUuid=dd44247c-6fba-49b4-a979-dafe76384873, infoPort=35291, infoSecurePort=0, ipcPort=35789, storageInfo=lv=-57;cid=testClusterID;nsid=1718931293;c=1731847751586), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:49:13,276 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x86e17bf191c349f7 with lease ID 0x7b0805c4811127: Processing first storage report for DS-3b93fa8b-6889-4be3-b82e-534af9454c07 from datanode DatanodeRegistration(127.0.0.1:39689, datanodeUuid=dd44247c-6fba-49b4-a979-dafe76384873, infoPort=35291, infoSecurePort=0, ipcPort=35789, storageInfo=lv=-57;cid=testClusterID;nsid=1718931293;c=1731847751586)
2024-11-17T12:49:13,276 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x86e17bf191c349f7 with lease ID 0x7b0805c4811127: from storage DS-3b93fa8b-6889-4be3-b82e-534af9454c07 node DatanodeRegistration(127.0.0.1:39689, datanodeUuid=dd44247c-6fba-49b4-a979-dafe76384873, infoPort=35291, infoSecurePort=0, ipcPort=35789, storageInfo=lv=-57;cid=testClusterID;nsid=1718931293;c=1731847751586), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:49:13,367 WARN  [Thread-1978 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/cluster_c97ab916-9cf5-6219-161a-81127106f55c/data/data3/current/BP-903435966-172.17.0.2-1731847751586/current, will proceed with Du for space computation calculation, 
2024-11-17T12:49:13,367 WARN  [Thread-1979 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/cluster_c97ab916-9cf5-6219-161a-81127106f55c/data/data4/current/BP-903435966-172.17.0.2-1731847751586/current, will proceed with Du for space computation calculation, 
2024-11-17T12:49:13,389 WARN  [Thread-1954 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T12:49:13,391 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4274b3ca7cb3c186 with lease ID 0x7b0805c4811128: Processing first storage report for DS-0e242a57-c9ac-4bbc-af96-7f585ff49a78 from datanode DatanodeRegistration(127.0.0.1:35075, datanodeUuid=1fa05234-3f75-49b6-b36d-70ec633312c1, infoPort=32955, infoSecurePort=0, ipcPort=38541, storageInfo=lv=-57;cid=testClusterID;nsid=1718931293;c=1731847751586)
2024-11-17T12:49:13,391 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4274b3ca7cb3c186 with lease ID 0x7b0805c4811128: from storage DS-0e242a57-c9ac-4bbc-af96-7f585ff49a78 node DatanodeRegistration(127.0.0.1:35075, datanodeUuid=1fa05234-3f75-49b6-b36d-70ec633312c1, infoPort=32955, infoSecurePort=0, ipcPort=38541, storageInfo=lv=-57;cid=testClusterID;nsid=1718931293;c=1731847751586), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:49:13,391 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4274b3ca7cb3c186 with lease ID 0x7b0805c4811128: Processing first storage report for DS-fd756209-bc46-438e-be5d-eafe3c64a986 from datanode DatanodeRegistration(127.0.0.1:35075, datanodeUuid=1fa05234-3f75-49b6-b36d-70ec633312c1, infoPort=32955, infoSecurePort=0, ipcPort=38541, storageInfo=lv=-57;cid=testClusterID;nsid=1718931293;c=1731847751586)
2024-11-17T12:49:13,391 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4274b3ca7cb3c186 with lease ID 0x7b0805c4811128: from storage DS-fd756209-bc46-438e-be5d-eafe3c64a986 node DatanodeRegistration(127.0.0.1:35075, datanodeUuid=1fa05234-3f75-49b6-b36d-70ec633312c1, infoPort=32955, infoSecurePort=0, ipcPort=38541, storageInfo=lv=-57;cid=testClusterID;nsid=1718931293;c=1731847751586), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:49:13,432 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29
2024-11-17T12:49:13,435 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/cluster_c97ab916-9cf5-6219-161a-81127106f55c/zookeeper_0, clientPort=61606, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/cluster_c97ab916-9cf5-6219-161a-81127106f55c/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/cluster_c97ab916-9cf5-6219-161a-81127106f55c/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-17T12:49:13,436 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61606
2024-11-17T12:49:13,436 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:49:13,438 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:49:13,453 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741825_1001 (size=7)
2024-11-17T12:49:13,453 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741825_1001 (size=7)
2024-11-17T12:49:13,454 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae with version=8
2024-11-17T12:49:13,454 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/hbase-staging
2024-11-17T12:49:13,457 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/0d751a622496:0 server-side Connection retries=45
2024-11-17T12:49:13,457 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:49:13,457 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T12:49:13,457 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T12:49:13,457 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:49:13,457 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T12:49:13,457 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-17T12:49:13,457 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T12:49:13,458 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42271
2024-11-17T12:49:13,460 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42271 connecting to ZooKeeper ensemble=127.0.0.1:61606
2024-11-17T12:49:13,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:422710x0, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T12:49:13,504 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42271-0x1014a21219e0000 connected
2024-11-17T12:49:13,590 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:49:13,592 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:49:13,594 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:49:13,594 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae, hbase.cluster.distributed=false
2024-11-17T12:49:13,595 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T12:49:13,596 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42271
2024-11-17T12:49:13,596 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42271
2024-11-17T12:49:13,596 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42271
2024-11-17T12:49:13,597 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42271
2024-11-17T12:49:13,597 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42271
2024-11-17T12:49:13,619 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/0d751a622496:0 server-side Connection retries=45
2024-11-17T12:49:13,619 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:49:13,619 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T12:49:13,620 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T12:49:13,620 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:49:13,620 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T12:49:13,620 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-17T12:49:13,620 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T12:49:13,621 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37879
2024-11-17T12:49:13,623 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37879 connecting to ZooKeeper ensemble=127.0.0.1:61606
2024-11-17T12:49:13,624 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:49:13,626 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:49:13,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:378790x0, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T12:49:13,640 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:378790x0, quorum=127.0.0.1:61606, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:49:13,640 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37879-0x1014a21219e0001 connected
2024-11-17T12:49:13,640 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-17T12:49:13,641 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-17T12:49:13,642 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37879-0x1014a21219e0001, quorum=127.0.0.1:61606, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-17T12:49:13,643 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37879-0x1014a21219e0001, quorum=127.0.0.1:61606, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T12:49:13,643 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37879
2024-11-17T12:49:13,643 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37879
2024-11-17T12:49:13,644 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37879
2024-11-17T12:49:13,651 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37879
2024-11-17T12:49:13,652 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37879
2024-11-17T12:49:13,665 DEBUG [M:0;0d751a622496:42271 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0d751a622496:42271
2024-11-17T12:49:13,666 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0d751a622496,42271,1731847753456
2024-11-17T12:49:13,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37879-0x1014a21219e0001, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:49:13,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:49:13,673 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0d751a622496,42271,1731847753456
2024-11-17T12:49:13,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37879-0x1014a21219e0001, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-17T12:49:13,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:13,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37879-0x1014a21219e0001, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:13,682 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-17T12:49:13,682 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0d751a622496,42271,1731847753456 from backup master directory
2024-11-17T12:49:13,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0d751a622496,42271,1731847753456
2024-11-17T12:49:13,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:49:13,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37879-0x1014a21219e0001, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:49:13,689 WARN  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T12:49:13,689 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0d751a622496,42271,1731847753456
2024-11-17T12:49:13,693 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/hbase.id] with ID: 486b3d2a-36b6-4273-b0b7-6743c02bc250
2024-11-17T12:49:13,694 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/.tmp/hbase.id
2024-11-17T12:49:13,699 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741826_1002 (size=42)
2024-11-17T12:49:13,700 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741826_1002 (size=42)
2024-11-17T12:49:13,700 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/.tmp/hbase.id]:[hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/hbase.id]
2024-11-17T12:49:13,714 INFO  [master/0d751a622496:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:49:13,714 INFO  [master/0d751a622496:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-17T12:49:13,716 INFO  [master/0d751a622496:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-17T12:49:13,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37879-0x1014a21219e0001, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:13,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:13,729 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741827_1003 (size=196)
2024-11-17T12:49:13,729 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741827_1003 (size=196)
2024-11-17T12:49:13,730 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-17T12:49:13,730 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-17T12:49:13,731 INFO  [master/0d751a622496:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T12:49:13,738 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741828_1004 (size=1189)
2024-11-17T12:49:13,738 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741828_1004 (size=1189)
2024-11-17T12:49:13,739 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/data/master/store
2024-11-17T12:49:13,746 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741829_1005 (size=34)
2024-11-17T12:49:13,746 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741829_1005 (size=34)
2024-11-17T12:49:13,746 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:49:13,746 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T12:49:13,747 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:49:13,747 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:49:13,747 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T12:49:13,747 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:49:13,747 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:49:13,747 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731847753746Disabling compacts and flushes for region at 1731847753746Disabling writes for close at 1731847753747 (+1 ms)Writing region close event to WAL at 1731847753747Closed at 1731847753747
2024-11-17T12:49:13,748 WARN  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/data/master/store/.initializing
2024-11-17T12:49:13,748 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/WALs/0d751a622496,42271,1731847753456
2024-11-17T12:49:13,750 INFO  [master/0d751a622496:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C42271%2C1731847753456, suffix=, logDir=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/WALs/0d751a622496,42271,1731847753456, archiveDir=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/oldWALs, maxLogs=10
2024-11-17T12:49:13,751 INFO  [master/0d751a622496:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C42271%2C1731847753456.1731847753750
2024-11-17T12:49:13,755 INFO  [master/0d751a622496:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/WALs/0d751a622496,42271,1731847753456/0d751a622496%2C42271%2C1731847753456.1731847753750
2024-11-17T12:49:13,756 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:32955:32955),(127.0.0.1/127.0.0.1:35291:35291)]
2024-11-17T12:49:13,757 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-17T12:49:13,757 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:49:13,757 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:49:13,757 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:49:13,758 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:49:13,760 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-17T12:49:13,760 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:13,761 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:49:13,761 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:49:13,762 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-17T12:49:13,762 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:13,762 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:49:13,763 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:49:13,764 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-17T12:49:13,764 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:13,765 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:49:13,765 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:49:13,766 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-17T12:49:13,767 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:13,767 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:49:13,767 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:49:13,768 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:49:13,769 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:49:13,770 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:49:13,770 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:49:13,771 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-17T12:49:13,773 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:49:13,775 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T12:49:13,776 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=800380, jitterRate=0.01773694157600403}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-17T12:49:13,777 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731847753757Initializing all the Stores at 1731847753758 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847753758Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847753758Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847753758Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847753758Cleaning up temporary data from old regions at 1731847753770 (+12 ms)Region opened successfully at 1731847753777 (+7 ms)
2024-11-17T12:49:13,777 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-17T12:49:13,781 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@121c54c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0d751a622496/172.17.0.2:0
2024-11-17T12:49:13,782 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-17T12:49:13,782 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-17T12:49:13,782 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-17T12:49:13,782 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-17T12:49:13,783 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-17T12:49:13,783 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-17T12:49:13,783 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-17T12:49:13,785 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-17T12:49:13,786 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-17T12:49:13,813 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-17T12:49:13,813 INFO  [master/0d751a622496:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-17T12:49:13,814 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-17T12:49:13,823 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-17T12:49:13,823 INFO  [master/0d751a622496:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-17T12:49:13,824 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-17T12:49:13,831 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-17T12:49:13,832 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-17T12:49:13,839 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-17T12:49:13,839 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:13,839 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:13,841 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-17T12:49:13,848 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-17T12:49:13,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37879-0x1014a21219e0001, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T12:49:13,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37879-0x1014a21219e0001, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:13,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T12:49:13,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:13,856 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0d751a622496,42271,1731847753456, sessionid=0x1014a21219e0000, setting cluster-up flag (Was=false)
2024-11-17T12:49:13,873 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37879-0x1014a21219e0001, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:13,873 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:13,898 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-17T12:49:13,899 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0d751a622496,42271,1731847753456
2024-11-17T12:49:13,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:13,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37879-0x1014a21219e0001, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:13,939 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-17T12:49:13,940 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0d751a622496,42271,1731847753456
2024-11-17T12:49:13,941 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-17T12:49:13,943 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-17T12:49:13,943 INFO  [master/0d751a622496:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-17T12:49:13,943 INFO  [master/0d751a622496:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-17T12:49:13,944 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0d751a622496,42271,1731847753456
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-17T12:49:13,945 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:49:13,945 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:49:13,945 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:49:13,945 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:49:13,945 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0d751a622496:0, corePoolSize=10, maxPoolSize=10
2024-11-17T12:49:13,945 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:49:13,945 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0d751a622496:0, corePoolSize=2, maxPoolSize=2
2024-11-17T12:49:13,945 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:49:13,947 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T12:49:13,947 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-17T12:49:13,948 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:13,948 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-17T12:49:13,949 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731847783949
2024-11-17T12:49:13,949 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-17T12:49:13,949 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-17T12:49:13,949 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-17T12:49:13,949 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-17T12:49:13,949 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-17T12:49:13,949 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-17T12:49:13,949 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:13,950 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-17T12:49:13,950 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-17T12:49:13,950 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-17T12:49:13,950 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-17T12:49:13,950 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-17T12:49:13,950 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.large.0-1731847753950,5,FailOnTimeoutGroup]
2024-11-17T12:49:13,951 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.small.0-1731847753951,5,FailOnTimeoutGroup]
2024-11-17T12:49:13,951 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:13,951 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-17T12:49:13,951 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:13,951 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:13,953 INFO  [RS:0;0d751a622496:37879 {}] regionserver.HRegionServer(746): ClusterId : 486b3d2a-36b6-4273-b0b7-6743c02bc250
2024-11-17T12:49:13,953 DEBUG [RS:0;0d751a622496:37879 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-17T12:49:13,954 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741831_1007 (size=1321)
2024-11-17T12:49:13,955 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741831_1007 (size=1321)
2024-11-17T12:49:13,955 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-17T12:49:13,956 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae
2024-11-17T12:49:13,961 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741832_1008 (size=32)
2024-11-17T12:49:13,961 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741832_1008 (size=32)
2024-11-17T12:49:13,962 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:49:13,963 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T12:49:13,964 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T12:49:13,964 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:13,965 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:49:13,965 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T12:49:13,965 DEBUG [RS:0;0d751a622496:37879 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-17T12:49:13,965 DEBUG [RS:0;0d751a622496:37879 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-17T12:49:13,967 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T12:49:13,967 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:13,967 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:49:13,967 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T12:49:13,968 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T12:49:13,968 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:13,969 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:49:13,969 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T12:49:13,970 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T12:49:13,970 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:13,970 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:49:13,971 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T12:49:13,971 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/hbase/meta/1588230740
2024-11-17T12:49:13,972 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/hbase/meta/1588230740
2024-11-17T12:49:13,973 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T12:49:13,973 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T12:49:13,973 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T12:49:13,974 DEBUG [RS:0;0d751a622496:37879 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-17T12:49:13,974 DEBUG [RS:0;0d751a622496:37879 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67efa6b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0d751a622496/172.17.0.2:0
2024-11-17T12:49:13,975 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T12:49:13,977 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T12:49:13,978 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=849987, jitterRate=0.0808149129152298}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T12:49:13,979 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731847753962Initializing all the Stores at 1731847753963 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847753963Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847753963Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847753963Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847753963Cleaning up temporary data from old regions at 1731847753973 (+10 ms)Region opened successfully at 1731847753979 (+6 ms)
2024-11-17T12:49:13,979 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T12:49:13,979 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T12:49:13,979 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T12:49:13,979 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T12:49:13,979 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T12:49:13,980 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T12:49:13,980 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731847753979Disabling compacts and flushes for region at 1731847753979Disabling writes for close at 1731847753979Writing region close event to WAL at 1731847753980 (+1 ms)Closed at 1731847753980
2024-11-17T12:49:13,991 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T12:49:13,991 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-17T12:49:13,991 DEBUG [RS:0;0d751a622496:37879 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0d751a622496:37879
2024-11-17T12:49:13,991 INFO  [RS:0;0d751a622496:37879 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-17T12:49:13,991 INFO  [RS:0;0d751a622496:37879 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-17T12:49:13,991 DEBUG [RS:0;0d751a622496:37879 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-17T12:49:13,991 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-17T12:49:13,992 INFO  [RS:0;0d751a622496:37879 {}] regionserver.HRegionServer(2659): reportForDuty to master=0d751a622496,42271,1731847753456 with port=37879, startcode=1731847753619
2024-11-17T12:49:13,992 DEBUG [RS:0;0d751a622496:37879 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-17T12:49:13,993 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T12:49:13,994 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-17T12:49:13,994 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36377, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService
2024-11-17T12:49:13,995 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42271 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0d751a622496,37879,1731847753619
2024-11-17T12:49:13,995 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42271 {}] master.ServerManager(517): Registering regionserver=0d751a622496,37879,1731847753619
2024-11-17T12:49:13,996 DEBUG [RS:0;0d751a622496:37879 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae
2024-11-17T12:49:13,996 DEBUG [RS:0;0d751a622496:37879 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41193
2024-11-17T12:49:13,996 DEBUG [RS:0;0d751a622496:37879 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-17T12:49:14,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T12:49:14,007 DEBUG [RS:0;0d751a622496:37879 {}] zookeeper.ZKUtil(111): regionserver:37879-0x1014a21219e0001, quorum=127.0.0.1:61606, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0d751a622496,37879,1731847753619
2024-11-17T12:49:14,007 WARN  [RS:0;0d751a622496:37879 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T12:49:14,007 INFO  [RS:0;0d751a622496:37879 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T12:49:14,007 DEBUG [RS:0;0d751a622496:37879 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/WALs/0d751a622496,37879,1731847753619
2024-11-17T12:49:14,011 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0d751a622496,37879,1731847753619]
2024-11-17T12:49:14,013 INFO  [RS:0;0d751a622496:37879 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-17T12:49:14,018 INFO  [RS:0;0d751a622496:37879 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-17T12:49:14,019 INFO  [RS:0;0d751a622496:37879 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-17T12:49:14,019 INFO  [RS:0;0d751a622496:37879 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:14,026 INFO  [RS:0;0d751a622496:37879 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-17T12:49:14,027 INFO  [RS:0;0d751a622496:37879 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-17T12:49:14,028 INFO  [RS:0;0d751a622496:37879 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:14,028 DEBUG [RS:0;0d751a622496:37879 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:49:14,028 DEBUG [RS:0;0d751a622496:37879 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:49:14,028 DEBUG [RS:0;0d751a622496:37879 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:49:14,028 DEBUG [RS:0;0d751a622496:37879 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:49:14,028 DEBUG [RS:0;0d751a622496:37879 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:49:14,028 DEBUG [RS:0;0d751a622496:37879 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0, corePoolSize=2, maxPoolSize=2
2024-11-17T12:49:14,028 DEBUG [RS:0;0d751a622496:37879 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:49:14,028 DEBUG [RS:0;0d751a622496:37879 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:49:14,028 DEBUG [RS:0;0d751a622496:37879 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:49:14,028 DEBUG [RS:0;0d751a622496:37879 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:49:14,028 DEBUG [RS:0;0d751a622496:37879 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:49:14,029 DEBUG [RS:0;0d751a622496:37879 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:49:14,029 DEBUG [RS:0;0d751a622496:37879 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0d751a622496:0, corePoolSize=3, maxPoolSize=3
2024-11-17T12:49:14,029 DEBUG [RS:0;0d751a622496:37879 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0, corePoolSize=3, maxPoolSize=3
2024-11-17T12:49:14,042 INFO  [RS:0;0d751a622496:37879 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:14,043 INFO  [RS:0;0d751a622496:37879 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:14,043 INFO  [RS:0;0d751a622496:37879 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:14,043 INFO  [RS:0;0d751a622496:37879 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:14,043 INFO  [RS:0;0d751a622496:37879 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:14,043 INFO  [RS:0;0d751a622496:37879 {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,37879,1731847753619-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T12:49:14,065 INFO  [RS:0;0d751a622496:37879 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-17T12:49:14,066 INFO  [RS:0;0d751a622496:37879 {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,37879,1731847753619-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:14,066 INFO  [RS:0;0d751a622496:37879 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:14,066 INFO  [RS:0;0d751a622496:37879 {}] regionserver.Replication(171): 0d751a622496,37879,1731847753619 started
2024-11-17T12:49:14,084 INFO  [RS:0;0d751a622496:37879 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:14,085 INFO  [RS:0;0d751a622496:37879 {}] regionserver.HRegionServer(1482): Serving as 0d751a622496,37879,1731847753619, RpcServer on 0d751a622496/172.17.0.2:37879, sessionid=0x1014a21219e0001
2024-11-17T12:49:14,085 DEBUG [RS:0;0d751a622496:37879 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-17T12:49:14,085 DEBUG [RS:0;0d751a622496:37879 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0d751a622496,37879,1731847753619
2024-11-17T12:49:14,085 DEBUG [RS:0;0d751a622496:37879 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0d751a622496,37879,1731847753619'
2024-11-17T12:49:14,085 DEBUG [RS:0;0d751a622496:37879 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-17T12:49:14,086 DEBUG [RS:0;0d751a622496:37879 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-17T12:49:14,086 DEBUG [RS:0;0d751a622496:37879 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-17T12:49:14,086 DEBUG [RS:0;0d751a622496:37879 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-17T12:49:14,086 DEBUG [RS:0;0d751a622496:37879 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0d751a622496,37879,1731847753619
2024-11-17T12:49:14,086 DEBUG [RS:0;0d751a622496:37879 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0d751a622496,37879,1731847753619'
2024-11-17T12:49:14,086 DEBUG [RS:0;0d751a622496:37879 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-17T12:49:14,086 DEBUG [RS:0;0d751a622496:37879 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-17T12:49:14,087 DEBUG [RS:0;0d751a622496:37879 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-17T12:49:14,087 INFO  [RS:0;0d751a622496:37879 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-17T12:49:14,087 INFO  [RS:0;0d751a622496:37879 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-17T12:49:14,144 WARN  [0d751a622496:42271 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-17T12:49:14,189 INFO  [RS:0;0d751a622496:37879 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C37879%2C1731847753619, suffix=, logDir=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/WALs/0d751a622496,37879,1731847753619, archiveDir=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/oldWALs, maxLogs=32
2024-11-17T12:49:14,189 INFO  [RS:0;0d751a622496:37879 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C37879%2C1731847753619.1731847754189
2024-11-17T12:49:14,199 INFO  [RS:0;0d751a622496:37879 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/WALs/0d751a622496,37879,1731847753619/0d751a622496%2C37879%2C1731847753619.1731847754189
2024-11-17T12:49:14,209 DEBUG [RS:0;0d751a622496:37879 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35291:35291),(127.0.0.1/127.0.0.1:32955:32955)]
2024-11-17T12:49:14,394 DEBUG [0d751a622496:42271 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-17T12:49:14,395 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0d751a622496,37879,1731847753619
2024-11-17T12:49:14,397 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0d751a622496,37879,1731847753619, state=OPENING
2024-11-17T12:49:14,455 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-17T12:49:14,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37879-0x1014a21219e0001, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:14,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:14,465 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:49:14,465 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:49:14,465 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T12:49:14,465 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0d751a622496,37879,1731847753619}]
2024-11-17T12:49:14,620 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-17T12:49:14,623 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39079, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-17T12:49:14,628 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-17T12:49:14,629 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T12:49:14,631 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C37879%2C1731847753619.meta, suffix=.meta, logDir=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/WALs/0d751a622496,37879,1731847753619, archiveDir=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/oldWALs, maxLogs=32
2024-11-17T12:49:14,631 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C37879%2C1731847753619.meta.1731847754631.meta
2024-11-17T12:49:14,638 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/WALs/0d751a622496,37879,1731847753619/0d751a622496%2C37879%2C1731847753619.meta.1731847754631.meta
2024-11-17T12:49:14,639 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:32955:32955),(127.0.0.1/127.0.0.1:35291:35291)]
2024-11-17T12:49:14,639 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-17T12:49:14,640 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-17T12:49:14,640 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-17T12:49:14,640 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-17T12:49:14,640 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-17T12:49:14,640 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:49:14,640 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-17T12:49:14,640 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-17T12:49:14,641 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T12:49:14,642 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T12:49:14,642 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:14,643 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:49:14,643 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T12:49:14,644 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T12:49:14,644 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:14,644 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:49:14,644 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T12:49:14,645 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T12:49:14,645 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:14,645 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:49:14,645 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T12:49:14,646 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T12:49:14,646 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:14,647 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:49:14,647 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T12:49:14,648 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/hbase/meta/1588230740
2024-11-17T12:49:14,649 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/hbase/meta/1588230740
2024-11-17T12:49:14,650 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T12:49:14,650 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T12:49:14,650 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T12:49:14,651 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T12:49:14,652 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=799148, jitterRate=0.01616986095905304}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T12:49:14,652 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-17T12:49:14,653 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731847754640Writing region info on filesystem at 1731847754640Initializing all the Stores at 1731847754641 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847754641Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847754641Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847754641Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847754641Cleaning up temporary data from old regions at 1731847754650 (+9 ms)Running coprocessor post-open hooks at 1731847754652 (+2 ms)Region opened successfully at 1731847754653 (+1 ms)
2024-11-17T12:49:14,654 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731847754619
2024-11-17T12:49:14,656 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-17T12:49:14,656 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-17T12:49:14,657 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0d751a622496,37879,1731847753619
2024-11-17T12:49:14,658 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0d751a622496,37879,1731847753619, state=OPEN
2024-11-17T12:49:14,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37879-0x1014a21219e0001, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T12:49:14,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T12:49:14,690 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0d751a622496,37879,1731847753619
2024-11-17T12:49:14,690 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:49:14,690 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:49:14,693 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-17T12:49:14,693 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0d751a622496,37879,1731847753619 in 225 msec
2024-11-17T12:49:14,697 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-17T12:49:14,697 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 703 msec
2024-11-17T12:49:14,697 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T12:49:14,697 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-17T12:49:14,699 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T12:49:14,699 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0d751a622496,37879,1731847753619, seqNum=-1]
2024-11-17T12:49:14,700 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T12:49:14,701 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39773, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T12:49:14,707 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 763 msec
2024-11-17T12:49:14,708 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731847754708, completionTime=-1
2024-11-17T12:49:14,708 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-17T12:49:14,708 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-17T12:49:14,710 INFO  [master/0d751a622496:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-17T12:49:14,710 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731847814710
2024-11-17T12:49:14,710 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731847874710
2024-11-17T12:49:14,710 INFO  [master/0d751a622496:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-11-17T12:49:14,710 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,42271,1731847753456-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:14,710 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,42271,1731847753456-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:14,710 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,42271,1731847753456-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:14,711 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0d751a622496:42271, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:14,711 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:14,711 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:14,713 DEBUG [master/0d751a622496:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-17T12:49:14,715 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.026sec
2024-11-17T12:49:14,716 INFO  [master/0d751a622496:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-17T12:49:14,716 INFO  [master/0d751a622496:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-17T12:49:14,716 INFO  [master/0d751a622496:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-17T12:49:14,716 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-17T12:49:14,716 INFO  [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-17T12:49:14,716 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,42271,1731847753456-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T12:49:14,716 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,42271,1731847753456-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-17T12:49:14,719 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-17T12:49:14,719 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-17T12:49:14,719 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,42271,1731847753456-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:14,754 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5164b8d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T12:49:14,754 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0d751a622496,42271,-1 for getting cluster id
2024-11-17T12:49:14,754 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-17T12:49:14,756 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '486b3d2a-36b6-4273-b0b7-6743c02bc250'
2024-11-17T12:49:14,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-17T12:49:14,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "486b3d2a-36b6-4273-b0b7-6743c02bc250"

2024-11-17T12:49:14,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cd00e9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T12:49:14,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0d751a622496,42271,-1]
2024-11-17T12:49:14,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-17T12:49:14,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:49:14,759 INFO  [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-17T12:49:14,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c6487af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T12:49:14,760 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T12:49:14,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0d751a622496,37879,1731847753619, seqNum=-1]
2024-11-17T12:49:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T12:49:14,763 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T12:49:14,766 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0d751a622496,42271,1731847753456
2024-11-17T12:49:14,766 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:49:14,769 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-17T12:49:14,769 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-17T12:49:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 0d751a622496,42271,1731847753456
2024-11-17T12:49:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3152bb03
2024-11-17T12:49:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-17T12:49:14,772 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-17T12:49:14,773 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-17T12:49:14,773 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-17T12:49:14,773 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-17T12:49:14,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling
2024-11-17T12:49:14,777 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-17T12:49:14,777 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:14,777 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRolling"
 procId is: 4
2024-11-17T12:49:14,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-17T12:49:14,778 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-17T12:49:14,785 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741835_1011 (size=381)
2024-11-17T12:49:14,785 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741835_1011 (size=381)
2024-11-17T12:49:14,787 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d6b23169599b74b84dd92cc0cf93118e, NAME => 'TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae
2024-11-17T12:49:14,794 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741836_1012 (size=64)
2024-11-17T12:49:14,794 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741836_1012 (size=64)
2024-11-17T12:49:14,795 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:49:14,795 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing d6b23169599b74b84dd92cc0cf93118e, disabling compactions & flushes
2024-11-17T12:49:14,795 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.
2024-11-17T12:49:14,795 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.
2024-11-17T12:49:14,795 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e. after waiting 0 ms
2024-11-17T12:49:14,795 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.
2024-11-17T12:49:14,795 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.
2024-11-17T12:49:14,795 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for d6b23169599b74b84dd92cc0cf93118e:
Waiting for close lock at 1731847754795Disabling compacts and flushes for region at 1731847754795Disabling writes for close at 1731847754795Writing region close event to WAL at 1731847754795Closed at 1731847754795
2024-11-17T12:49:14,796 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-17T12:49:14,797 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731847754796"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731847754796"}]},"ts":"1731847754796"}
2024-11-17T12:49:14,799 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-17T12:49:14,801 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-17T12:49:14,801 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731847754801"}]},"ts":"1731847754801"}
2024-11-17T12:49:14,803 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta
2024-11-17T12:49:14,804 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d6b23169599b74b84dd92cc0cf93118e, ASSIGN}]
2024-11-17T12:49:14,805 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d6b23169599b74b84dd92cc0cf93118e, ASSIGN
2024-11-17T12:49:14,806 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d6b23169599b74b84dd92cc0cf93118e, ASSIGN; state=OFFLINE, location=0d751a622496,37879,1731847753619; forceNewPlan=false, retain=false
2024-11-17T12:49:14,840 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:14,840 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:14,957 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d6b23169599b74b84dd92cc0cf93118e, regionState=OPENING, regionLocation=0d751a622496,37879,1731847753619
2024-11-17T12:49:14,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d6b23169599b74b84dd92cc0cf93118e, ASSIGN because future has completed
2024-11-17T12:49:14,960 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6b23169599b74b84dd92cc0cf93118e, server=0d751a622496,37879,1731847753619}]
2024-11-17T12:49:15,118 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.
2024-11-17T12:49:15,118 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => d6b23169599b74b84dd92cc0cf93118e, NAME => 'TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.', STARTKEY => '', ENDKEY => ''}
2024-11-17T12:49:15,119 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:15,119 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:49:15,119 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:15,119 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:15,121 INFO  [StoreOpener-d6b23169599b74b84dd92cc0cf93118e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region d6b23169599b74b84dd92cc0cf93118e 
2024-11-17T12:49:15,122 INFO  [StoreOpener-d6b23169599b74b84dd92cc0cf93118e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6b23169599b74b84dd92cc0cf93118e columnFamilyName info
2024-11-17T12:49:15,122 DEBUG [StoreOpener-d6b23169599b74b84dd92cc0cf93118e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:15,123 INFO  [StoreOpener-d6b23169599b74b84dd92cc0cf93118e-1 {}] regionserver.HStore(327): Store=d6b23169599b74b84dd92cc0cf93118e/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:49:15,123 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:15,125 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:15,125 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:15,126 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:15,126 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:15,127 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:15,129 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T12:49:15,129 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened d6b23169599b74b84dd92cc0cf93118e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=706341, jitterRate=-0.10184118151664734}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-17T12:49:15,130 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:15,130 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for d6b23169599b74b84dd92cc0cf93118e:
Running coprocessor pre-open hook at 1731847755119Writing region info on filesystem at 1731847755119Initializing all the Stores at 1731847755120 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847755120Cleaning up temporary data from old regions at 1731847755126 (+6 ms)Running coprocessor post-open hooks at 1731847755130 (+4 ms)Region opened successfully at 1731847755130
2024-11-17T12:49:15,131 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e., pid=6, masterSystemTime=1731847755113
2024-11-17T12:49:15,133 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.
2024-11-17T12:49:15,133 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.
2024-11-17T12:49:15,134 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d6b23169599b74b84dd92cc0cf93118e, regionState=OPEN, openSeqNum=2, regionLocation=0d751a622496,37879,1731847753619
2024-11-17T12:49:15,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6b23169599b74b84dd92cc0cf93118e, server=0d751a622496,37879,1731847753619 because future has completed
2024-11-17T12:49:15,140 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-17T12:49:15,140 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d6b23169599b74b84dd92cc0cf93118e, server=0d751a622496,37879,1731847753619 in 178 msec
2024-11-17T12:49:15,143 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-17T12:49:15,143 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d6b23169599b74b84dd92cc0cf93118e, ASSIGN in 336 msec
2024-11-17T12:49:15,145 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-17T12:49:15,145 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731847755145"}]},"ts":"1731847755145"}
2024-11-17T12:49:15,147 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta
2024-11-17T12:49:15,148 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-17T12:49:15,150 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 374 msec
2024-11-17T12:49:15,339 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,339 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,339 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,339 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,339 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,339 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,340 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,340 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,363 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,363 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,363 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,363 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,363 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,364 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,367 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,367 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,367 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,370 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,840 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:15,840 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:15,874 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-17T12:49:15,877 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,877 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,878 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,878 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,878 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,878 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,879 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,879 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,902 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,903 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,903 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,903 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,903 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,904 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,909 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,910 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,910 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:15,914 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:16,841 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:16,841 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:17,842 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:17,842 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:18,842 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:18,842 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:19,843 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:19,843 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:20,013 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-17T12:49:20,014 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling'
2024-11-17T12:49:20,479 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-17T12:49:20,480 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-17T12:49:20,481 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T12:49:20,481 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-17T12:49:20,481 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-11-17T12:49:20,481 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-17T12:49:20,844 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:20,844 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:21,844 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:21,844 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:22,845 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:22,845 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:23,846 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:23,846 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:24,847 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:24,847 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-17T12:49:24,850 INFO  [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed
2024-11-17T12:49:24,850 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100
2024-11-17T12:49:24,854 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling
2024-11-17T12:49:24,854 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.
2024-11-17T12:49:24,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e., hostname=0d751a622496,37879,1731847753619, seqNum=2]
2024-11-17T12:49:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37879 {}] regionserver.HRegion(8855): Flush requested on d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:24,876 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6b23169599b74b84dd92cc0cf93118e 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-17T12:49:24,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/d55101adc775466a97ce30d63f975f8e is 1080, key is row0001/info:/1731847764859/Put/seqid=0
2024-11-17T12:49:24,902 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741837_1013 (size=12509)
2024-11-17T12:49:24,903 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741837_1013 (size=12509)
2024-11-17T12:49:24,904 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/d55101adc775466a97ce30d63f975f8e
2024-11-17T12:49:24,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/d55101adc775466a97ce30d63f975f8e as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/d55101adc775466a97ce30d63f975f8e
2024-11-17T12:49:24,916 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/d55101adc775466a97ce30d63f975f8e, entries=7, sequenceid=11, filesize=12.2 K
2024-11-17T12:49:24,917 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=18.91 KB/19368 for d6b23169599b74b84dd92cc0cf93118e in 41ms, sequenceid=11, compaction requested=false
2024-11-17T12:49:24,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6b23169599b74b84dd92cc0cf93118e:

2024-11-17T12:49:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37879 {}] regionserver.HRegion(8855): Flush requested on d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:24,920 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6b23169599b74b84dd92cc0cf93118e 1/1 column families, dataSize=19.96 KB heapSize=21.63 KB
2024-11-17T12:49:24,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/f38a405920e94ce69bf37a78813ae403 is 1080, key is row0008/info:/1731847764877/Put/seqid=0
2024-11-17T12:49:24,929 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741838_1014 (size=25453)
2024-11-17T12:49:24,930 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741838_1014 (size=25453)
2024-11-17T12:49:24,930 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=19.96 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/f38a405920e94ce69bf37a78813ae403
2024-11-17T12:49:24,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/f38a405920e94ce69bf37a78813ae403 as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/f38a405920e94ce69bf37a78813ae403
2024-11-17T12:49:24,942 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/f38a405920e94ce69bf37a78813ae403, entries=19, sequenceid=33, filesize=24.9 K
2024-11-17T12:49:24,943 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~19.96 KB/20444, heapSize ~21.61 KB/22128, currentSize=6.30 KB/6456 for d6b23169599b74b84dd92cc0cf93118e in 23ms, sequenceid=33, compaction requested=false
2024-11-17T12:49:24,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6b23169599b74b84dd92cc0cf93118e:

2024-11-17T12:49:24,943 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=37.1 K, sizeToCheck=16.0 K
2024-11-17T12:49:24,943 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:49:24,943 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/f38a405920e94ce69bf37a78813ae403 because midkey is the same as first or last row
2024-11-17T12:49:25,847 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:25,847 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:26,848 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:26,848 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37879 {}] regionserver.HRegion(8855): Flush requested on d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:26,937 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6b23169599b74b84dd92cc0cf93118e 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-17T12:49:26,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/7d5499ad4abc4a35af43de444543d147 is 1080, key is row0027/info:/1731847764921/Put/seqid=0
2024-11-17T12:49:26,949 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741839_1015 (size=12509)
2024-11-17T12:49:26,949 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741839_1015 (size=12509)
2024-11-17T12:49:26,950 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/7d5499ad4abc4a35af43de444543d147
2024-11-17T12:49:26,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/7d5499ad4abc4a35af43de444543d147 as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/7d5499ad4abc4a35af43de444543d147
2024-11-17T12:49:26,963 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/7d5499ad4abc4a35af43de444543d147, entries=7, sequenceid=43, filesize=12.2 K
2024-11-17T12:49:26,964 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=10.51 KB/10760 for d6b23169599b74b84dd92cc0cf93118e in 27ms, sequenceid=43, compaction requested=true
2024-11-17T12:49:26,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6b23169599b74b84dd92cc0cf93118e:

2024-11-17T12:49:26,964 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=49.3 K, sizeToCheck=16.0 K
2024-11-17T12:49:26,964 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:49:26,965 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/f38a405920e94ce69bf37a78813ae403 because midkey is the same as first or last row
2024-11-17T12:49:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37879 {}] regionserver.HRegion(8855): Flush requested on d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:26,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6b23169599b74b84dd92cc0cf93118e:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T12:49:26,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:49:26,965 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6b23169599b74b84dd92cc0cf93118e 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-17T12:49:26,965 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T12:49:26,966 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 50471 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-17T12:49:26,966 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1541): d6b23169599b74b84dd92cc0cf93118e/info is initiating minor compaction (all files)
2024-11-17T12:49:26,966 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6b23169599b74b84dd92cc0cf93118e/info in TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.
2024-11-17T12:49:26,967 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/d55101adc775466a97ce30d63f975f8e, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/f38a405920e94ce69bf37a78813ae403, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/7d5499ad4abc4a35af43de444543d147] into tmpdir=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp, totalSize=49.3 K
2024-11-17T12:49:26,967 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting d55101adc775466a97ce30d63f975f8e, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731847764859
2024-11-17T12:49:26,968 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting f38a405920e94ce69bf37a78813ae403, keycount=19, bloomtype=ROW, size=24.9 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731847764877
2024-11-17T12:49:26,968 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d5499ad4abc4a35af43de444543d147, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731847764921
2024-11-17T12:49:26,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/2510cf4d99534de7bb7a046c88571a55 is 1080, key is row0034/info:/1731847766939/Put/seqid=0
2024-11-17T12:49:26,980 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741840_1016 (size=16817)
2024-11-17T12:49:26,980 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741840_1016 (size=16817)
2024-11-17T12:49:26,981 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/2510cf4d99534de7bb7a046c88571a55
2024-11-17T12:49:26,987 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6b23169599b74b84dd92cc0cf93118e#info#compaction#59 average throughput is 16.93 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T12:49:26,988 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/d7dab8f2cf964bee9e3ec9c1f032b2db is 1080, key is row0001/info:/1731847764859/Put/seqid=0
2024-11-17T12:49:26,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/2510cf4d99534de7bb7a046c88571a55 as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/2510cf4d99534de7bb7a046c88571a55
2024-11-17T12:49:26,998 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/2510cf4d99534de7bb7a046c88571a55, entries=11, sequenceid=57, filesize=16.4 K
2024-11-17T12:49:26,999 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=13.66 KB/13988 for d6b23169599b74b84dd92cc0cf93118e in 34ms, sequenceid=57, compaction requested=false
2024-11-17T12:49:26,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6b23169599b74b84dd92cc0cf93118e:

2024-11-17T12:49:26,999 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=65.7 K, sizeToCheck=16.0 K
2024-11-17T12:49:26,999 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:49:26,999 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/f38a405920e94ce69bf37a78813ae403 because midkey is the same as first or last row
2024-11-17T12:49:27,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37879 {}] regionserver.HRegion(8855): Flush requested on d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:27,001 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6b23169599b74b84dd92cc0cf93118e 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-11-17T12:49:27,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/ff4705b0b26a456a86df86ed6b82ca3e is 1080, key is row0045/info:/1731847766966/Put/seqid=0
2024-11-17T12:49:27,020 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741841_1017 (size=40670)
2024-11-17T12:49:27,020 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741841_1017 (size=40670)
2024-11-17T12:49:27,028 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/d7dab8f2cf964bee9e3ec9c1f032b2db as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/d7dab8f2cf964bee9e3ec9c1f032b2db
2024-11-17T12:49:27,031 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741842_1018 (size=20064)
2024-11-17T12:49:27,031 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741842_1018 (size=20064)
2024-11-17T12:49:27,032 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/ff4705b0b26a456a86df86ed6b82ca3e
2024-11-17T12:49:27,035 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6b23169599b74b84dd92cc0cf93118e/info of d6b23169599b74b84dd92cc0cf93118e into d7dab8f2cf964bee9e3ec9c1f032b2db(size=39.7 K), total size for store is 56.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T12:49:27,035 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6b23169599b74b84dd92cc0cf93118e:

2024-11-17T12:49:27,035 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e., storeName=d6b23169599b74b84dd92cc0cf93118e/info, priority=13, startTime=1731847766965; duration=0sec
2024-11-17T12:49:27,035 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=56.1 K, sizeToCheck=16.0 K
2024-11-17T12:49:27,035 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:49:27,035 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/d7dab8f2cf964bee9e3ec9c1f032b2db because midkey is the same as first or last row
2024-11-17T12:49:27,035 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=56.1 K, sizeToCheck=16.0 K
2024-11-17T12:49:27,035 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:49:27,035 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/d7dab8f2cf964bee9e3ec9c1f032b2db because midkey is the same as first or last row
2024-11-17T12:49:27,035 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=56.1 K, sizeToCheck=16.0 K
2024-11-17T12:49:27,035 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:49:27,035 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/d7dab8f2cf964bee9e3ec9c1f032b2db because midkey is the same as first or last row
2024-11-17T12:49:27,035 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:49:27,035 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6b23169599b74b84dd92cc0cf93118e:info
2024-11-17T12:49:27,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/ff4705b0b26a456a86df86ed6b82ca3e as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/ff4705b0b26a456a86df86ed6b82ca3e
2024-11-17T12:49:27,044 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/ff4705b0b26a456a86df86ed6b82ca3e, entries=14, sequenceid=74, filesize=19.6 K
2024-11-17T12:49:27,045 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=6.30 KB/6456 for d6b23169599b74b84dd92cc0cf93118e in 44ms, sequenceid=74, compaction requested=true
2024-11-17T12:49:27,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6b23169599b74b84dd92cc0cf93118e:

2024-11-17T12:49:27,045 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=75.7 K, sizeToCheck=16.0 K
2024-11-17T12:49:27,045 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:49:27,045 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/d7dab8f2cf964bee9e3ec9c1f032b2db because midkey is the same as first or last row
2024-11-17T12:49:27,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6b23169599b74b84dd92cc0cf93118e:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T12:49:27,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:49:27,045 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T12:49:27,047 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 77551 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-17T12:49:27,047 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1541): d6b23169599b74b84dd92cc0cf93118e/info is initiating minor compaction (all files)
2024-11-17T12:49:27,047 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6b23169599b74b84dd92cc0cf93118e/info in TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.
2024-11-17T12:49:27,047 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/d7dab8f2cf964bee9e3ec9c1f032b2db, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/2510cf4d99534de7bb7a046c88571a55, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/ff4705b0b26a456a86df86ed6b82ca3e] into tmpdir=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp, totalSize=75.7 K
2024-11-17T12:49:27,047 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7dab8f2cf964bee9e3ec9c1f032b2db, keycount=33, bloomtype=ROW, size=39.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731847764859
2024-11-17T12:49:27,047 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2510cf4d99534de7bb7a046c88571a55, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1731847766939
2024-11-17T12:49:27,048 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff4705b0b26a456a86df86ed6b82ca3e, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731847766966
2024-11-17T12:49:27,060 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6b23169599b74b84dd92cc0cf93118e#info#compaction#61 average throughput is 19.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T12:49:27,060 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/1bca6eecb22f4c8a9c14957f03cf26db is 1080, key is row0001/info:/1731847764859/Put/seqid=0
2024-11-17T12:49:27,066 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741843_1019 (size=67766)
2024-11-17T12:49:27,066 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741843_1019 (size=67766)
2024-11-17T12:49:27,072 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/1bca6eecb22f4c8a9c14957f03cf26db as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/1bca6eecb22f4c8a9c14957f03cf26db
2024-11-17T12:49:27,079 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6b23169599b74b84dd92cc0cf93118e/info of d6b23169599b74b84dd92cc0cf93118e into 1bca6eecb22f4c8a9c14957f03cf26db(size=66.2 K), total size for store is 66.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T12:49:27,079 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6b23169599b74b84dd92cc0cf93118e:

2024-11-17T12:49:27,079 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e., storeName=d6b23169599b74b84dd92cc0cf93118e/info, priority=13, startTime=1731847767045; duration=0sec
2024-11-17T12:49:27,079 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=66.2 K, sizeToCheck=16.0 K
2024-11-17T12:49:27,079 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:49:27,079 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/1bca6eecb22f4c8a9c14957f03cf26db because midkey is the same as first or last row
2024-11-17T12:49:27,079 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=66.2 K, sizeToCheck=16.0 K
2024-11-17T12:49:27,079 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:49:27,079 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/1bca6eecb22f4c8a9c14957f03cf26db because midkey is the same as first or last row
2024-11-17T12:49:27,079 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=66.2 K, sizeToCheck=16.0 K
2024-11-17T12:49:27,079 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:49:27,079 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/1bca6eecb22f4c8a9c14957f03cf26db because midkey is the same as first or last row
2024-11-17T12:49:27,079 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:49:27,080 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6b23169599b74b84dd92cc0cf93118e:info
2024-11-17T12:49:27,849 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:27,849 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:28,849 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:28,849 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:29,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37879 {}] regionserver.HRegion(8855): Flush requested on d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:29,018 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6b23169599b74b84dd92cc0cf93118e 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-17T12:49:29,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/6fce7f8c26b24ea9b0ae858c87035b1a is 1080, key is row0059/info:/1731847767002/Put/seqid=0
2024-11-17T12:49:29,033 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741844_1020 (size=12509)
2024-11-17T12:49:29,033 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/6fce7f8c26b24ea9b0ae858c87035b1a
2024-11-17T12:49:29,034 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741844_1020 (size=12509)
2024-11-17T12:49:29,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/6fce7f8c26b24ea9b0ae858c87035b1a as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/6fce7f8c26b24ea9b0ae858c87035b1a
2024-11-17T12:49:29,046 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/6fce7f8c26b24ea9b0ae858c87035b1a, entries=7, sequenceid=86, filesize=12.2 K
2024-11-17T12:49:29,047 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=12.61 KB/12912 for d6b23169599b74b84dd92cc0cf93118e in 29ms, sequenceid=86, compaction requested=false
2024-11-17T12:49:29,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6b23169599b74b84dd92cc0cf93118e:

2024-11-17T12:49:29,047 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=78.4 K, sizeToCheck=16.0 K
2024-11-17T12:49:29,047 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:49:29,047 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/1bca6eecb22f4c8a9c14957f03cf26db because midkey is the same as first or last row
2024-11-17T12:49:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37879 {}] regionserver.HRegion(8855): Flush requested on d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:29,048 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6b23169599b74b84dd92cc0cf93118e 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-17T12:49:29,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/e6b9dd7234fd4c4baed298ef2c9ce74d is 1080, key is row0066/info:/1731847769019/Put/seqid=0
2024-11-17T12:49:29,057 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741845_1021 (size=18987)
2024-11-17T12:49:29,058 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741845_1021 (size=18987)
2024-11-17T12:49:29,058 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/e6b9dd7234fd4c4baed298ef2c9ce74d
2024-11-17T12:49:29,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/e6b9dd7234fd4c4baed298ef2c9ce74d as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/e6b9dd7234fd4c4baed298ef2c9ce74d
2024-11-17T12:49:29,071 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/e6b9dd7234fd4c4baed298ef2c9ce74d, entries=13, sequenceid=102, filesize=18.5 K
2024-11-17T12:49:29,072 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=10.51 KB/10760 for d6b23169599b74b84dd92cc0cf93118e in 24ms, sequenceid=102, compaction requested=true
2024-11-17T12:49:29,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6b23169599b74b84dd92cc0cf93118e:

2024-11-17T12:49:29,072 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=96.9 K, sizeToCheck=16.0 K
2024-11-17T12:49:29,072 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:49:29,072 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/1bca6eecb22f4c8a9c14957f03cf26db because midkey is the same as first or last row
2024-11-17T12:49:29,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d6b23169599b74b84dd92cc0cf93118e:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T12:49:29,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:49:29,072 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T12:49:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37879 {}] regionserver.HRegion(8855): Flush requested on d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:29,072 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d6b23169599b74b84dd92cc0cf93118e 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-17T12:49:29,073 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 99262 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-17T12:49:29,073 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1541): d6b23169599b74b84dd92cc0cf93118e/info is initiating minor compaction (all files)
2024-11-17T12:49:29,074 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d6b23169599b74b84dd92cc0cf93118e/info in TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.
2024-11-17T12:49:29,074 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/1bca6eecb22f4c8a9c14957f03cf26db, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/6fce7f8c26b24ea9b0ae858c87035b1a, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/e6b9dd7234fd4c4baed298ef2c9ce74d] into tmpdir=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp, totalSize=96.9 K
2024-11-17T12:49:29,074 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bca6eecb22f4c8a9c14957f03cf26db, keycount=58, bloomtype=ROW, size=66.2 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731847764859
2024-11-17T12:49:29,075 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fce7f8c26b24ea9b0ae858c87035b1a, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731847767002
2024-11-17T12:49:29,075 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6b9dd7234fd4c4baed298ef2c9ce74d, keycount=13, bloomtype=ROW, size=18.5 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731847769019
2024-11-17T12:49:29,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/3624f13c2494474fa7b8747dcff910ec is 1080, key is row0079/info:/1731847769049/Put/seqid=0
2024-11-17T12:49:29,082 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741846_1022 (size=16817)
2024-11-17T12:49:29,082 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741846_1022 (size=16817)
2024-11-17T12:49:29,083 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/3624f13c2494474fa7b8747dcff910ec
2024-11-17T12:49:29,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/3624f13c2494474fa7b8747dcff910ec as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/3624f13c2494474fa7b8747dcff910ec
2024-11-17T12:49:29,092 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d6b23169599b74b84dd92cc0cf93118e#info#compaction#65 average throughput is 26.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T12:49:29,093 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/be7b1ecc88d647b18ec40ad9f2004c7f is 1080, key is row0001/info:/1731847764859/Put/seqid=0
2024-11-17T12:49:29,096 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741847_1023 (size=89485)
2024-11-17T12:49:29,096 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741847_1023 (size=89485)
2024-11-17T12:49:29,097 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/3624f13c2494474fa7b8747dcff910ec, entries=11, sequenceid=116, filesize=16.4 K
2024-11-17T12:49:29,098 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=7.36 KB/7532 for d6b23169599b74b84dd92cc0cf93118e in 26ms, sequenceid=116, compaction requested=false
2024-11-17T12:49:29,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d6b23169599b74b84dd92cc0cf93118e:

2024-11-17T12:49:29,098 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=113.4 K, sizeToCheck=16.0 K
2024-11-17T12:49:29,098 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:49:29,098 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/1bca6eecb22f4c8a9c14957f03cf26db because midkey is the same as first or last row
2024-11-17T12:49:29,102 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/be7b1ecc88d647b18ec40ad9f2004c7f as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/be7b1ecc88d647b18ec40ad9f2004c7f
2024-11-17T12:49:29,108 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d6b23169599b74b84dd92cc0cf93118e/info of d6b23169599b74b84dd92cc0cf93118e into be7b1ecc88d647b18ec40ad9f2004c7f(size=87.4 K), total size for store is 103.8 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T12:49:29,108 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d6b23169599b74b84dd92cc0cf93118e:

2024-11-17T12:49:29,108 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e., storeName=d6b23169599b74b84dd92cc0cf93118e/info, priority=13, startTime=1731847769072; duration=0sec
2024-11-17T12:49:29,108 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=103.8 K, sizeToCheck=16.0 K
2024-11-17T12:49:29,108 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:49:29,109 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=103.8 K, sizeToCheck=16.0 K
2024-11-17T12:49:29,109 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:49:29,109 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=103.8 K, sizeToCheck=16.0 K
2024-11-17T12:49:29,109 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-17T12:49:29,110 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:49:29,110 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:49:29,110 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d6b23169599b74b84dd92cc0cf93118e:info
2024-11-17T12:49:29,111 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42271 {}] assignment.AssignmentManager(1363): Split request from 0d751a622496,37879,1731847753619, parent={ENCODED => d6b23169599b74b84dd92cc0cf93118e, NAME => 'TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-11-17T12:49:29,116 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42271 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=0d751a622496,37879,1731847753619
2024-11-17T12:49:29,119 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42271 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=d6b23169599b74b84dd92cc0cf93118e, daughterA=e36f819d8f0d43e1647618946ce7d017, daughterB=8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:29,121 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=d6b23169599b74b84dd92cc0cf93118e, daughterA=e36f819d8f0d43e1647618946ce7d017, daughterB=8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:29,121 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=d6b23169599b74b84dd92cc0cf93118e, daughterA=e36f819d8f0d43e1647618946ce7d017, daughterB=8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:29,121 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=d6b23169599b74b84dd92cc0cf93118e, daughterA=e36f819d8f0d43e1647618946ce7d017, daughterB=8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:29,129 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d6b23169599b74b84dd92cc0cf93118e, UNASSIGN}]
2024-11-17T12:49:29,130 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d6b23169599b74b84dd92cc0cf93118e, UNASSIGN
2024-11-17T12:49:29,132 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=d6b23169599b74b84dd92cc0cf93118e, regionState=CLOSING, regionLocation=0d751a622496,37879,1731847753619
2024-11-17T12:49:29,135 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d6b23169599b74b84dd92cc0cf93118e, UNASSIGN because future has completed
2024-11-17T12:49:29,135 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false
2024-11-17T12:49:29,136 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure d6b23169599b74b84dd92cc0cf93118e, server=0d751a622496,37879,1731847753619}]
2024-11-17T12:49:29,294 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:29,294 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true
2024-11-17T12:49:29,295 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing d6b23169599b74b84dd92cc0cf93118e, disabling compactions & flushes
2024-11-17T12:49:29,295 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.
2024-11-17T12:49:29,295 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.
2024-11-17T12:49:29,295 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e. after waiting 0 ms
2024-11-17T12:49:29,295 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.
2024-11-17T12:49:29,295 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(2902): Flushing d6b23169599b74b84dd92cc0cf93118e 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-17T12:49:29,301 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/67427ed6f2ae40e99e14ff9908858a0c is 1080, key is row0090/info:/1731847769074/Put/seqid=0
2024-11-17T12:49:29,307 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741848_1024 (size=12509)
2024-11-17T12:49:29,308 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741848_1024 (size=12509)
2024-11-17T12:49:29,308 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/67427ed6f2ae40e99e14ff9908858a0c
2024-11-17T12:49:29,315 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/.tmp/info/67427ed6f2ae40e99e14ff9908858a0c as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/67427ed6f2ae40e99e14ff9908858a0c
2024-11-17T12:49:29,321 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/67427ed6f2ae40e99e14ff9908858a0c, entries=7, sequenceid=127, filesize=12.2 K
2024-11-17T12:49:29,322 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for d6b23169599b74b84dd92cc0cf93118e in 27ms, sequenceid=127, compaction requested=true
2024-11-17T12:49:29,324 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/d55101adc775466a97ce30d63f975f8e, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/f38a405920e94ce69bf37a78813ae403, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/d7dab8f2cf964bee9e3ec9c1f032b2db, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/7d5499ad4abc4a35af43de444543d147, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/2510cf4d99534de7bb7a046c88571a55, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/1bca6eecb22f4c8a9c14957f03cf26db, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/ff4705b0b26a456a86df86ed6b82ca3e, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/6fce7f8c26b24ea9b0ae858c87035b1a, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/e6b9dd7234fd4c4baed298ef2c9ce74d] to archive
2024-11-17T12:49:29,325 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-17T12:49:29,327 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/d55101adc775466a97ce30d63f975f8e to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/d55101adc775466a97ce30d63f975f8e
2024-11-17T12:49:29,328 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/f38a405920e94ce69bf37a78813ae403 to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/f38a405920e94ce69bf37a78813ae403
2024-11-17T12:49:29,330 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/d7dab8f2cf964bee9e3ec9c1f032b2db to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/d7dab8f2cf964bee9e3ec9c1f032b2db
2024-11-17T12:49:29,331 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/7d5499ad4abc4a35af43de444543d147 to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/7d5499ad4abc4a35af43de444543d147
2024-11-17T12:49:29,332 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/2510cf4d99534de7bb7a046c88571a55 to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/2510cf4d99534de7bb7a046c88571a55
2024-11-17T12:49:29,334 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/1bca6eecb22f4c8a9c14957f03cf26db to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/1bca6eecb22f4c8a9c14957f03cf26db
2024-11-17T12:49:29,335 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/ff4705b0b26a456a86df86ed6b82ca3e to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/ff4705b0b26a456a86df86ed6b82ca3e
2024-11-17T12:49:29,336 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/6fce7f8c26b24ea9b0ae858c87035b1a to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/6fce7f8c26b24ea9b0ae858c87035b1a
2024-11-17T12:49:29,338 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/e6b9dd7234fd4c4baed298ef2c9ce74d to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/e6b9dd7234fd4c4baed298ef2c9ce74d
2024-11-17T12:49:29,344 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=1
2024-11-17T12:49:29,345 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.
2024-11-17T12:49:29,345 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for d6b23169599b74b84dd92cc0cf93118e:
Waiting for close lock at 1731847769295Running coprocessor pre-close hooks at 1731847769295Disabling compacts and flushes for region at 1731847769295Disabling writes for close at 1731847769295Obtaining lock to block concurrent updates at 1731847769295Preparing flush snapshotting stores in d6b23169599b74b84dd92cc0cf93118e at 1731847769295Finished memstore snapshotting TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e., syncing WAL and waiting on mvcc, flushsize=dataSize=7532, getHeapSize=8304, getOffHeapSize=0, getCellsCount=7 at 1731847769296 (+1 ms)Flushing stores of TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e. at 1731847769297 (+1 ms)Flushing d6b23169599b74b84dd92cc0cf93118e/info: creating writer at 1731847769297Flushing d6b23169599b74b84dd92cc0cf93118e/info: appending metadata at 1731847769300 (+3 ms)Flushing d6b23169599b74b84dd92cc0cf93118e/info: closing flushed file at 1731847769301 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6eca41d: reopening flushed file at 1731847769314 (+13 ms)Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for d6b23169599b74b84dd92cc0cf93118e in 27ms, sequenceid=127, compaction requested=true at 1731847769322 (+8 ms)Writing region close event to WAL at 1731847769340 (+18 ms)Running coprocessor post-close hooks at 1731847769345 (+5 ms)Closed at 1731847769345
2024-11-17T12:49:29,347 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:29,348 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=d6b23169599b74b84dd92cc0cf93118e, regionState=CLOSED
2024-11-17T12:49:29,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure d6b23169599b74b84dd92cc0cf93118e, server=0d751a622496,37879,1731847753619 because future has completed
2024-11-17T12:49:29,354 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8
2024-11-17T12:49:29,354 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure d6b23169599b74b84dd92cc0cf93118e, server=0d751a622496,37879,1731847753619 in 217 msec
2024-11-17T12:49:29,356 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-17T12:49:29,356 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=d6b23169599b74b84dd92cc0cf93118e, UNASSIGN in 226 msec
2024-11-17T12:49:29,364 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:29,367 INFO  [PEWorker-4 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 3 storefiles, region=d6b23169599b74b84dd92cc0cf93118e, threads=3
2024-11-17T12:49:29,370 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/3624f13c2494474fa7b8747dcff910ec for region: d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:29,370 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/67427ed6f2ae40e99e14ff9908858a0c for region: d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:29,370 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/be7b1ecc88d647b18ec40ad9f2004c7f for region: d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:29,380 DEBUG [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/67427ed6f2ae40e99e14ff9908858a0c, top=true
2024-11-17T12:49:29,380 DEBUG [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/3624f13c2494474fa7b8747dcff910ec, top=true
2024-11-17T12:49:29,395 INFO  [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/TestLogRolling-testLogRolling=d6b23169599b74b84dd92cc0cf93118e-67427ed6f2ae40e99e14ff9908858a0c for child: 8f60fdcd6b99d651c379bb67f8bffc8b, parent: d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:29,396 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/67427ed6f2ae40e99e14ff9908858a0c for region: d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:29,396 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741849_1025 (size=27)
2024-11-17T12:49:29,396 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741849_1025 (size=27)
2024-11-17T12:49:29,403 INFO  [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/TestLogRolling-testLogRolling=d6b23169599b74b84dd92cc0cf93118e-3624f13c2494474fa7b8747dcff910ec for child: 8f60fdcd6b99d651c379bb67f8bffc8b, parent: d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:29,403 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/3624f13c2494474fa7b8747dcff910ec for region: d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:29,411 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741850_1026 (size=27)
2024-11-17T12:49:29,411 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741850_1026 (size=27)
2024-11-17T12:49:29,411 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/be7b1ecc88d647b18ec40ad9f2004c7f for region: d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:29,413 DEBUG [PEWorker-4 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region d6b23169599b74b84dd92cc0cf93118e Daughter A: [hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/e36f819d8f0d43e1647618946ce7d017/info/be7b1ecc88d647b18ec40ad9f2004c7f.d6b23169599b74b84dd92cc0cf93118e] storefiles, Daughter B: [hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/TestLogRolling-testLogRolling=d6b23169599b74b84dd92cc0cf93118e-3624f13c2494474fa7b8747dcff910ec, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/TestLogRolling-testLogRolling=d6b23169599b74b84dd92cc0cf93118e-67427ed6f2ae40e99e14ff9908858a0c, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/be7b1ecc88d647b18ec40ad9f2004c7f.d6b23169599b74b84dd92cc0cf93118e] storefiles.
2024-11-17T12:49:29,420 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741851_1027 (size=71)
2024-11-17T12:49:29,422 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741851_1027 (size=71)
2024-11-17T12:49:29,424 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:29,437 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741852_1028 (size=71)
2024-11-17T12:49:29,437 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741852_1028 (size=71)
2024-11-17T12:49:29,439 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:29,450 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/e36f819d8f0d43e1647618946ce7d017/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1
2024-11-17T12:49:29,452 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1
2024-11-17T12:49:29,455 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731847769454"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1731847769454"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1731847769454"}]},"ts":"1731847769454"}
2024-11-17T12:49:29,455 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1731847769116.e36f819d8f0d43e1647618946ce7d017.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731847769454"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731847769454"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1731847769454"}]},"ts":"1731847769454"}
2024-11-17T12:49:29,455 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731847769454"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731847769454"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1731847769454"}]},"ts":"1731847769454"}
2024-11-17T12:49:29,473 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e36f819d8f0d43e1647618946ce7d017, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8f60fdcd6b99d651c379bb67f8bffc8b, ASSIGN}]
2024-11-17T12:49:29,474 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8f60fdcd6b99d651c379bb67f8bffc8b, ASSIGN
2024-11-17T12:49:29,474 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e36f819d8f0d43e1647618946ce7d017, ASSIGN
2024-11-17T12:49:29,475 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e36f819d8f0d43e1647618946ce7d017, ASSIGN; state=SPLITTING_NEW, location=0d751a622496,37879,1731847753619; forceNewPlan=false, retain=false
2024-11-17T12:49:29,475 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8f60fdcd6b99d651c379bb67f8bffc8b, ASSIGN; state=SPLITTING_NEW, location=0d751a622496,37879,1731847753619; forceNewPlan=false, retain=false
2024-11-17T12:49:29,626 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e36f819d8f0d43e1647618946ce7d017, regionState=OPENING, regionLocation=0d751a622496,37879,1731847753619
2024-11-17T12:49:29,626 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=8f60fdcd6b99d651c379bb67f8bffc8b, regionState=OPENING, regionLocation=0d751a622496,37879,1731847753619
2024-11-17T12:49:29,628 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e36f819d8f0d43e1647618946ce7d017, ASSIGN because future has completed
2024-11-17T12:49:29,629 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure e36f819d8f0d43e1647618946ce7d017, server=0d751a622496,37879,1731847753619}]
2024-11-17T12:49:29,630 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8f60fdcd6b99d651c379bb67f8bffc8b, ASSIGN because future has completed
2024-11-17T12:49:29,630 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f60fdcd6b99d651c379bb67f8bffc8b, server=0d751a622496,37879,1731847753619}]
2024-11-17T12:49:29,785 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1731847769116.e36f819d8f0d43e1647618946ce7d017.
2024-11-17T12:49:29,785 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => e36f819d8f0d43e1647618946ce7d017, NAME => 'TestLogRolling-testLogRolling,,1731847769116.e36f819d8f0d43e1647618946ce7d017.', STARTKEY => '', ENDKEY => 'row0062'}
2024-11-17T12:49:29,785 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling e36f819d8f0d43e1647618946ce7d017
2024-11-17T12:49:29,785 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731847769116.e36f819d8f0d43e1647618946ce7d017.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:49:29,785 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for e36f819d8f0d43e1647618946ce7d017
2024-11-17T12:49:29,785 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for e36f819d8f0d43e1647618946ce7d017
2024-11-17T12:49:29,787 INFO  [StoreOpener-e36f819d8f0d43e1647618946ce7d017-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region e36f819d8f0d43e1647618946ce7d017 
2024-11-17T12:49:29,787 INFO  [StoreOpener-e36f819d8f0d43e1647618946ce7d017-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e36f819d8f0d43e1647618946ce7d017 columnFamilyName info
2024-11-17T12:49:29,787 DEBUG [StoreOpener-e36f819d8f0d43e1647618946ce7d017-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:29,799 DEBUG [StoreOpener-e36f819d8f0d43e1647618946ce7d017-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/e36f819d8f0d43e1647618946ce7d017/info/be7b1ecc88d647b18ec40ad9f2004c7f.d6b23169599b74b84dd92cc0cf93118e->hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/be7b1ecc88d647b18ec40ad9f2004c7f-bottom
2024-11-17T12:49:29,800 INFO  [StoreOpener-e36f819d8f0d43e1647618946ce7d017-1 {}] regionserver.HStore(327): Store=e36f819d8f0d43e1647618946ce7d017/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:49:29,800 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for e36f819d8f0d43e1647618946ce7d017
2024-11-17T12:49:29,801 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/e36f819d8f0d43e1647618946ce7d017
2024-11-17T12:49:29,802 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/e36f819d8f0d43e1647618946ce7d017
2024-11-17T12:49:29,803 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for e36f819d8f0d43e1647618946ce7d017
2024-11-17T12:49:29,803 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for e36f819d8f0d43e1647618946ce7d017
2024-11-17T12:49:29,805 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for e36f819d8f0d43e1647618946ce7d017
2024-11-17T12:49:29,806 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened e36f819d8f0d43e1647618946ce7d017; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=795104, jitterRate=0.011027097702026367}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-17T12:49:29,806 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e36f819d8f0d43e1647618946ce7d017
2024-11-17T12:49:29,807 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for e36f819d8f0d43e1647618946ce7d017:
Running coprocessor pre-open hook at 1731847769785Writing region info on filesystem at 1731847769785Initializing all the Stores at 1731847769786 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847769786Cleaning up temporary data from old regions at 1731847769803 (+17 ms)Running coprocessor post-open hooks at 1731847769806 (+3 ms)Region opened successfully at 1731847769807 (+1 ms)
2024-11-17T12:49:29,808 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1731847769116.e36f819d8f0d43e1647618946ce7d017., pid=12, masterSystemTime=1731847769781
2024-11-17T12:49:29,808 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store e36f819d8f0d43e1647618946ce7d017:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T12:49:29,808 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:49:29,808 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-11-17T12:49:29,809 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1731847769116.e36f819d8f0d43e1647618946ce7d017.
2024-11-17T12:49:29,809 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1541): e36f819d8f0d43e1647618946ce7d017/info is initiating minor compaction (all files)
2024-11-17T12:49:29,809 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e36f819d8f0d43e1647618946ce7d017/info in TestLogRolling-testLogRolling,,1731847769116.e36f819d8f0d43e1647618946ce7d017.
2024-11-17T12:49:29,809 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/e36f819d8f0d43e1647618946ce7d017/info/be7b1ecc88d647b18ec40ad9f2004c7f.d6b23169599b74b84dd92cc0cf93118e->hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/be7b1ecc88d647b18ec40ad9f2004c7f-bottom] into tmpdir=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/e36f819d8f0d43e1647618946ce7d017/.tmp, totalSize=87.4 K
2024-11-17T12:49:29,810 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting be7b1ecc88d647b18ec40ad9f2004c7f.d6b23169599b74b84dd92cc0cf93118e, keycount=39, bloomtype=ROW, size=87.4 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731847764859
2024-11-17T12:49:29,810 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1731847769116.e36f819d8f0d43e1647618946ce7d017.
2024-11-17T12:49:29,810 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1731847769116.e36f819d8f0d43e1647618946ce7d017.
2024-11-17T12:49:29,811 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.
2024-11-17T12:49:29,811 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f60fdcd6b99d651c379bb67f8bffc8b, NAME => 'TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.', STARTKEY => 'row0062', ENDKEY => ''}
2024-11-17T12:49:29,811 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:29,811 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:49:29,811 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:29,811 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:29,811 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e36f819d8f0d43e1647618946ce7d017, regionState=OPEN, openSeqNum=131, regionLocation=0d751a622496,37879,1731847753619
2024-11-17T12:49:29,813 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37879 {}] regionserver.HRegion(8855): Flush requested on 1588230740
2024-11-17T12:49:29,813 INFO  [StoreOpener-8f60fdcd6b99d651c379bb67f8bffc8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 8f60fdcd6b99d651c379bb67f8bffc8b 
2024-11-17T12:49:29,813 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-11-17T12:49:29,813 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.11 KB heapSize=8.96 KB
2024-11-17T12:49:29,813 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure e36f819d8f0d43e1647618946ce7d017, server=0d751a622496,37879,1731847753619 because future has completed
2024-11-17T12:49:29,815 INFO  [StoreOpener-8f60fdcd6b99d651c379bb67f8bffc8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f60fdcd6b99d651c379bb67f8bffc8b columnFamilyName info
2024-11-17T12:49:29,815 DEBUG [StoreOpener-8f60fdcd6b99d651c379bb67f8bffc8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:29,823 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10
2024-11-17T12:49:29,823 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure e36f819d8f0d43e1647618946ce7d017, server=0d751a622496,37879,1731847753619 in 186 msec
2024-11-17T12:49:29,825 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e36f819d8f0d43e1647618946ce7d017, ASSIGN in 350 msec
2024-11-17T12:49:29,833 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e36f819d8f0d43e1647618946ce7d017#info#compaction#67 average throughput is 20.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T12:49:29,833 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/e36f819d8f0d43e1647618946ce7d017/.tmp/info/aead052a27fd4afc9282dcd21739ec3d is 1080, key is row0001/info:/1731847764859/Put/seqid=0
2024-11-17T12:49:29,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/hbase/meta/1588230740/.tmp/info/2ed7820ee1e046c1bdead76aa60650ac is 193, key is TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b./info:regioninfo/1731847769626/Put/seqid=0
2024-11-17T12:49:29,838 DEBUG [StoreOpener-8f60fdcd6b99d651c379bb67f8bffc8b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/TestLogRolling-testLogRolling=d6b23169599b74b84dd92cc0cf93118e-3624f13c2494474fa7b8747dcff910ec
2024-11-17T12:49:29,842 DEBUG [StoreOpener-8f60fdcd6b99d651c379bb67f8bffc8b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/TestLogRolling-testLogRolling=d6b23169599b74b84dd92cc0cf93118e-67427ed6f2ae40e99e14ff9908858a0c
2024-11-17T12:49:29,847 DEBUG [StoreOpener-8f60fdcd6b99d651c379bb67f8bffc8b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/be7b1ecc88d647b18ec40ad9f2004c7f.d6b23169599b74b84dd92cc0cf93118e->hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/be7b1ecc88d647b18ec40ad9f2004c7f-top
2024-11-17T12:49:29,848 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741853_1029 (size=70862)
2024-11-17T12:49:29,848 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741853_1029 (size=70862)
2024-11-17T12:49:29,848 INFO  [StoreOpener-8f60fdcd6b99d651c379bb67f8bffc8b-1 {}] regionserver.HStore(327): Store=8f60fdcd6b99d651c379bb67f8bffc8b/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:49:29,848 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:29,849 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:29,850 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:29,851 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741854_1030 (size=9847)
2024-11-17T12:49:29,850 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:29,850 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:29,851 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741854_1030 (size=9847)
2024-11-17T12:49:29,851 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:29,851 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:29,851 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.92 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/hbase/meta/1588230740/.tmp/info/2ed7820ee1e046c1bdead76aa60650ac
2024-11-17T12:49:29,853 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:29,854 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 8f60fdcd6b99d651c379bb67f8bffc8b; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=826957, jitterRate=0.051530271768569946}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-17T12:49:29,854 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:29,854 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 8f60fdcd6b99d651c379bb67f8bffc8b:
Running coprocessor pre-open hook at 1731847769811Writing region info on filesystem at 1731847769811Initializing all the Stores at 1731847769813 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847769813Cleaning up temporary data from old regions at 1731847769851 (+38 ms)Running coprocessor post-open hooks at 1731847769854 (+3 ms)Region opened successfully at 1731847769854
2024-11-17T12:49:29,856 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b., pid=13, masterSystemTime=1731847769781
2024-11-17T12:49:29,856 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store 8f60fdcd6b99d651c379bb67f8bffc8b:info, priority=-2147483648, current under compaction store size is 2
2024-11-17T12:49:29,856 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0
2024-11-17T12:49:29,856 DEBUG [RS:0;0d751a622496:37879-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T12:49:29,857 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/e36f819d8f0d43e1647618946ce7d017/.tmp/info/aead052a27fd4afc9282dcd21739ec3d as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/e36f819d8f0d43e1647618946ce7d017/info/aead052a27fd4afc9282dcd21739ec3d
2024-11-17T12:49:29,858 INFO  [RS:0;0d751a622496:37879-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.
2024-11-17T12:49:29,858 DEBUG [RS:0;0d751a622496:37879-longCompactions-0 {}] regionserver.HStore(1541): 8f60fdcd6b99d651c379bb67f8bffc8b/info is initiating minor compaction (all files)
2024-11-17T12:49:29,859 INFO  [RS:0;0d751a622496:37879-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f60fdcd6b99d651c379bb67f8bffc8b/info in TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.
2024-11-17T12:49:29,859 INFO  [RS:0;0d751a622496:37879-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/be7b1ecc88d647b18ec40ad9f2004c7f.d6b23169599b74b84dd92cc0cf93118e->hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/be7b1ecc88d647b18ec40ad9f2004c7f-top, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/TestLogRolling-testLogRolling=d6b23169599b74b84dd92cc0cf93118e-3624f13c2494474fa7b8747dcff910ec, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/TestLogRolling-testLogRolling=d6b23169599b74b84dd92cc0cf93118e-67427ed6f2ae40e99e14ff9908858a0c] into tmpdir=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp, totalSize=116.0 K
2024-11-17T12:49:29,859 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.
2024-11-17T12:49:29,859 INFO  [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.
2024-11-17T12:49:29,859 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=8f60fdcd6b99d651c379bb67f8bffc8b, regionState=OPEN, openSeqNum=131, regionLocation=0d751a622496,37879,1731847753619
2024-11-17T12:49:29,861 DEBUG [RS:0;0d751a622496:37879-longCompactions-0 {}] compactions.Compactor(225): Compacting be7b1ecc88d647b18ec40ad9f2004c7f.d6b23169599b74b84dd92cc0cf93118e, keycount=39, bloomtype=ROW, size=87.4 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731847764859
2024-11-17T12:49:29,861 DEBUG [RS:0;0d751a622496:37879-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=d6b23169599b74b84dd92cc0cf93118e-3624f13c2494474fa7b8747dcff910ec, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731847769049
2024-11-17T12:49:29,861 DEBUG [RS:0;0d751a622496:37879-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=d6b23169599b74b84dd92cc0cf93118e-67427ed6f2ae40e99e14ff9908858a0c, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731847769074
2024-11-17T12:49:29,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f60fdcd6b99d651c379bb67f8bffc8b, server=0d751a622496,37879,1731847753619 because future has completed
2024-11-17T12:49:29,866 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in e36f819d8f0d43e1647618946ce7d017/info of e36f819d8f0d43e1647618946ce7d017 into aead052a27fd4afc9282dcd21739ec3d(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T12:49:29,866 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=11
2024-11-17T12:49:29,866 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e36f819d8f0d43e1647618946ce7d017:

2024-11-17T12:49:29,866 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 8f60fdcd6b99d651c379bb67f8bffc8b, server=0d751a622496,37879,1731847753619 in 233 msec
2024-11-17T12:49:29,866 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731847769116.e36f819d8f0d43e1647618946ce7d017., storeName=e36f819d8f0d43e1647618946ce7d017/info, priority=15, startTime=1731847769808; duration=0sec
2024-11-17T12:49:29,866 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:49:29,866 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e36f819d8f0d43e1647618946ce7d017:info
2024-11-17T12:49:29,869 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=7
2024-11-17T12:49:29,869 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8f60fdcd6b99d651c379bb67f8bffc8b, ASSIGN in 393 msec
2024-11-17T12:49:29,872 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=d6b23169599b74b84dd92cc0cf93118e, daughterA=e36f819d8f0d43e1647618946ce7d017, daughterB=8f60fdcd6b99d651c379bb67f8bffc8b in 753 msec
2024-11-17T12:49:29,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/hbase/meta/1588230740/.tmp/ns/31e510b87bb44d28a016f847d058cd99 is 43, key is default/ns:d/1731847754701/Put/seqid=0
2024-11-17T12:49:29,889 INFO  [RS:0;0d751a622496:37879-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f60fdcd6b99d651c379bb67f8bffc8b#info#compaction#70 average throughput is 17.96 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T12:49:29,890 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741855_1031 (size=5153)
2024-11-17T12:49:29,889 DEBUG [RS:0;0d751a622496:37879-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/14c2b5c8ed0a4b73964a6e08b8a4fd7d is 1080, key is row0062/info:/1731847767010/Put/seqid=0
2024-11-17T12:49:29,890 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741855_1031 (size=5153)
2024-11-17T12:49:29,890 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/hbase/meta/1588230740/.tmp/ns/31e510b87bb44d28a016f847d058cd99
2024-11-17T12:49:29,895 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741856_1032 (size=42984)
2024-11-17T12:49:29,896 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741856_1032 (size=42984)
2024-11-17T12:49:29,903 DEBUG [RS:0;0d751a622496:37879-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/14c2b5c8ed0a4b73964a6e08b8a4fd7d as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/14c2b5c8ed0a4b73964a6e08b8a4fd7d
2024-11-17T12:49:29,909 INFO  [RS:0;0d751a622496:37879-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f60fdcd6b99d651c379bb67f8bffc8b/info of 8f60fdcd6b99d651c379bb67f8bffc8b into 14c2b5c8ed0a4b73964a6e08b8a4fd7d(size=42.0 K), total size for store is 42.0 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T12:49:29,909 DEBUG [RS:0;0d751a622496:37879-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f60fdcd6b99d651c379bb67f8bffc8b:

2024-11-17T12:49:29,909 INFO  [RS:0;0d751a622496:37879-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b., storeName=8f60fdcd6b99d651c379bb67f8bffc8b/info, priority=13, startTime=1731847769856; duration=0sec
2024-11-17T12:49:29,910 DEBUG [RS:0;0d751a622496:37879-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:49:29,910 DEBUG [RS:0;0d751a622496:37879-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f60fdcd6b99d651c379bb67f8bffc8b:info
2024-11-17T12:49:29,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/hbase/meta/1588230740/.tmp/table/401ed37c43e549ddb291d242ba78321a is 65, key is TestLogRolling-testLogRolling/table:state/1731847755145/Put/seqid=0
2024-11-17T12:49:29,921 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741857_1033 (size=5340)
2024-11-17T12:49:29,921 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741857_1033 (size=5340)
2024-11-17T12:49:29,921 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/hbase/meta/1588230740/.tmp/table/401ed37c43e549ddb291d242ba78321a
2024-11-17T12:49:29,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/hbase/meta/1588230740/.tmp/info/2ed7820ee1e046c1bdead76aa60650ac as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/hbase/meta/1588230740/info/2ed7820ee1e046c1bdead76aa60650ac
2024-11-17T12:49:29,932 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/hbase/meta/1588230740/info/2ed7820ee1e046c1bdead76aa60650ac, entries=30, sequenceid=17, filesize=9.6 K
2024-11-17T12:49:29,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/hbase/meta/1588230740/.tmp/ns/31e510b87bb44d28a016f847d058cd99 as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/hbase/meta/1588230740/ns/31e510b87bb44d28a016f847d058cd99
2024-11-17T12:49:29,938 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/hbase/meta/1588230740/ns/31e510b87bb44d28a016f847d058cd99, entries=2, sequenceid=17, filesize=5.0 K
2024-11-17T12:49:29,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/hbase/meta/1588230740/.tmp/table/401ed37c43e549ddb291d242ba78321a as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/hbase/meta/1588230740/table/401ed37c43e549ddb291d242ba78321a
2024-11-17T12:49:29,944 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/hbase/meta/1588230740/table/401ed37c43e549ddb291d242ba78321a, entries=2, sequenceid=17, filesize=5.2 K
2024-11-17T12:49:29,945 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.11 KB/5234, heapSize ~8.66 KB/8872, currentSize=705 B/705 for 1588230740 in 132ms, sequenceid=17, compaction requested=false
2024-11-17T12:49:29,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-17T12:49:30,851 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:30,851 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37879 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:59156 deadline: 1731847781092, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e. is not online on 0d751a622496,37879,1731847753619
2024-11-17T12:49:31,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e., hostname=0d751a622496,37879,1731847753619, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e., hostname=0d751a622496,37879,1731847753619, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e. is not online on 0d751a622496,37879,1731847753619
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-17T12:49:31,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e., hostname=0d751a622496,37879,1731847753619, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e. is not online on 0d751a622496,37879,1731847753619
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-17T12:49:31,121 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1731847754773.d6b23169599b74b84dd92cc0cf93118e., hostname=0d751a622496,37879,1731847753619, seqNum=2 from cache
2024-11-17T12:49:31,852 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:31,852 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:32,852 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:32,852 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:33,853 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:33,853 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:34,347 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,347 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,347 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,347 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,347 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,347 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,348 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,348 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,373 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,373 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,373 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,374 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,374 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,374 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,378 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,378 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,378 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,380 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,854 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:34,854 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:34,887 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-17T12:49:34,889 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,889 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,890 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,890 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,890 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,890 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,891 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,891 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,917 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,918 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,918 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,918 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,918 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,919 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,924 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,924 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,924 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:34,928 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-17T12:49:35,854 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:35,854 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:36,855 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:36,855 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:37,856 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:37,856 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:38,857 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:38,857 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:39,858 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:39,858 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:40,859 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:40,859 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:41,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0097', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b., hostname=0d751a622496,37879,1731847753619, seqNum=131]
2024-11-17T12:49:41,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37879 {}] regionserver.HRegion(8855): Flush requested on 8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:41,260 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f60fdcd6b99d651c379bb67f8bffc8b 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-17T12:49:41,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/0e24fdbe21d1492faea2a4dc40c6382b is 1080, key is row0097/info:/1731847781247/Put/seqid=0
2024-11-17T12:49:41,270 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741858_1034 (size=12516)
2024-11-17T12:49:41,270 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741858_1034 (size=12516)
2024-11-17T12:49:41,271 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/0e24fdbe21d1492faea2a4dc40c6382b
2024-11-17T12:49:41,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/0e24fdbe21d1492faea2a4dc40c6382b as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/0e24fdbe21d1492faea2a4dc40c6382b
2024-11-17T12:49:41,286 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/0e24fdbe21d1492faea2a4dc40c6382b, entries=7, sequenceid=141, filesize=12.2 K
2024-11-17T12:49:41,287 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for 8f60fdcd6b99d651c379bb67f8bffc8b in 27ms, sequenceid=141, compaction requested=false
2024-11-17T12:49:41,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f60fdcd6b99d651c379bb67f8bffc8b:

2024-11-17T12:49:41,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37879 {}] regionserver.HRegion(8855): Flush requested on 8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:41,288 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f60fdcd6b99d651c379bb67f8bffc8b 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-17T12:49:41,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/9f6e33cb33854f0ca5fb5d739d0e928c is 1080, key is row0104/info:/1731847781261/Put/seqid=0
2024-11-17T12:49:41,298 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741859_1035 (size=17906)
2024-11-17T12:49:41,298 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741859_1035 (size=17906)
2024-11-17T12:49:41,298 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/9f6e33cb33854f0ca5fb5d739d0e928c
2024-11-17T12:49:41,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/9f6e33cb33854f0ca5fb5d739d0e928c as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/9f6e33cb33854f0ca5fb5d739d0e928c
2024-11-17T12:49:41,314 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/9f6e33cb33854f0ca5fb5d739d0e928c, entries=12, sequenceid=156, filesize=17.5 K
2024-11-17T12:49:41,315 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=12.61 KB/12912 for 8f60fdcd6b99d651c379bb67f8bffc8b in 26ms, sequenceid=156, compaction requested=true
2024-11-17T12:49:41,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f60fdcd6b99d651c379bb67f8bffc8b:

2024-11-17T12:49:41,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f60fdcd6b99d651c379bb67f8bffc8b:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T12:49:41,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:49:41,315 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T12:49:41,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37879 {}] regionserver.HRegion(8855): Flush requested on 8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:41,316 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 73406 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-17T12:49:41,316 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1541): 8f60fdcd6b99d651c379bb67f8bffc8b/info is initiating minor compaction (all files)
2024-11-17T12:49:41,316 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f60fdcd6b99d651c379bb67f8bffc8b 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-17T12:49:41,316 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f60fdcd6b99d651c379bb67f8bffc8b/info in TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.
2024-11-17T12:49:41,316 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/14c2b5c8ed0a4b73964a6e08b8a4fd7d, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/0e24fdbe21d1492faea2a4dc40c6382b, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/9f6e33cb33854f0ca5fb5d739d0e928c] into tmpdir=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp, totalSize=71.7 K
2024-11-17T12:49:41,317 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14c2b5c8ed0a4b73964a6e08b8a4fd7d, keycount=35, bloomtype=ROW, size=42.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731847767010
2024-11-17T12:49:41,317 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e24fdbe21d1492faea2a4dc40c6382b, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731847781247
2024-11-17T12:49:41,317 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f6e33cb33854f0ca5fb5d739d0e928c, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731847781261
2024-11-17T12:49:41,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/ed1d2fe5e9d74c47954103cce804338d is 1080, key is row0116/info:/1731847781289/Put/seqid=0
2024-11-17T12:49:41,331 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741860_1036 (size=19000)
2024-11-17T12:49:41,332 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741860_1036 (size=19000)
2024-11-17T12:49:41,332 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/ed1d2fe5e9d74c47954103cce804338d
2024-11-17T12:49:41,335 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f60fdcd6b99d651c379bb67f8bffc8b#info#compaction#75 average throughput is 55.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T12:49:41,336 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/8890a648fe534f77a2bbb6d150235bbe is 1080, key is row0062/info:/1731847767010/Put/seqid=0
2024-11-17T12:49:41,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/ed1d2fe5e9d74c47954103cce804338d as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/ed1d2fe5e9d74c47954103cce804338d
2024-11-17T12:49:41,342 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741861_1037 (size=63636)
2024-11-17T12:49:41,342 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741861_1037 (size=63636)
2024-11-17T12:49:41,346 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/ed1d2fe5e9d74c47954103cce804338d, entries=13, sequenceid=172, filesize=18.6 K
2024-11-17T12:49:41,347 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=0 B/0 for 8f60fdcd6b99d651c379bb67f8bffc8b in 31ms, sequenceid=172, compaction requested=false
2024-11-17T12:49:41,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f60fdcd6b99d651c379bb67f8bffc8b:

2024-11-17T12:49:41,349 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/8890a648fe534f77a2bbb6d150235bbe as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/8890a648fe534f77a2bbb6d150235bbe
2024-11-17T12:49:41,354 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f60fdcd6b99d651c379bb67f8bffc8b/info of 8f60fdcd6b99d651c379bb67f8bffc8b into 8890a648fe534f77a2bbb6d150235bbe(size=62.1 K), total size for store is 80.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T12:49:41,354 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f60fdcd6b99d651c379bb67f8bffc8b:

2024-11-17T12:49:41,355 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b., storeName=8f60fdcd6b99d651c379bb67f8bffc8b/info, priority=13, startTime=1731847781315; duration=0sec
2024-11-17T12:49:41,355 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:49:41,355 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f60fdcd6b99d651c379bb67f8bffc8b:info
2024-11-17T12:49:41,859 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:41,859 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:42,860 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:42,860 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:43,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37879 {}] regionserver.HRegion(8855): Flush requested on 8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:43,334 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f60fdcd6b99d651c379bb67f8bffc8b 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-17T12:49:43,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/10bdead0bdce4550a644275e6fff4c06 is 1080, key is row0129/info:/1731847783318/Put/seqid=0
2024-11-17T12:49:43,344 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741862_1038 (size=12516)
2024-11-17T12:49:43,344 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741862_1038 (size=12516)
2024-11-17T12:49:43,344 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/10bdead0bdce4550a644275e6fff4c06
2024-11-17T12:49:43,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/10bdead0bdce4550a644275e6fff4c06 as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/10bdead0bdce4550a644275e6fff4c06
2024-11-17T12:49:43,357 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/10bdead0bdce4550a644275e6fff4c06, entries=7, sequenceid=183, filesize=12.2 K
2024-11-17T12:49:43,358 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for 8f60fdcd6b99d651c379bb67f8bffc8b in 25ms, sequenceid=183, compaction requested=true
2024-11-17T12:49:43,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f60fdcd6b99d651c379bb67f8bffc8b:

2024-11-17T12:49:43,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f60fdcd6b99d651c379bb67f8bffc8b:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T12:49:43,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:49:43,359 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T12:49:43,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37879 {}] regionserver.HRegion(8855): Flush requested on 8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:43,359 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f60fdcd6b99d651c379bb67f8bffc8b 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-17T12:49:43,360 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 95152 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-17T12:49:43,360 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1541): 8f60fdcd6b99d651c379bb67f8bffc8b/info is initiating minor compaction (all files)
2024-11-17T12:49:43,360 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f60fdcd6b99d651c379bb67f8bffc8b/info in TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.
2024-11-17T12:49:43,360 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/8890a648fe534f77a2bbb6d150235bbe, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/ed1d2fe5e9d74c47954103cce804338d, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/10bdead0bdce4550a644275e6fff4c06] into tmpdir=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp, totalSize=92.9 K
2024-11-17T12:49:43,361 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8890a648fe534f77a2bbb6d150235bbe, keycount=54, bloomtype=ROW, size=62.1 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731847767010
2024-11-17T12:49:43,361 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed1d2fe5e9d74c47954103cce804338d, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731847781289
2024-11-17T12:49:43,361 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10bdead0bdce4550a644275e6fff4c06, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731847783318
2024-11-17T12:49:43,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/3fc4e4fd62dd4142941b1fd6ee4c44ea is 1080, key is row0136/info:/1731847783335/Put/seqid=0
2024-11-17T12:49:43,368 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741863_1039 (size=17906)
2024-11-17T12:49:43,368 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741863_1039 (size=17906)
2024-11-17T12:49:43,369 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/3fc4e4fd62dd4142941b1fd6ee4c44ea
2024-11-17T12:49:43,374 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f60fdcd6b99d651c379bb67f8bffc8b#info#compaction#78 average throughput is 37.97 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T12:49:43,375 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/f7905615b95148a1b0cc749bba05030b is 1080, key is row0062/info:/1731847767010/Put/seqid=0
2024-11-17T12:49:43,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/3fc4e4fd62dd4142941b1fd6ee4c44ea as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/3fc4e4fd62dd4142941b1fd6ee4c44ea
2024-11-17T12:49:43,383 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741864_1040 (size=85371)
2024-11-17T12:49:43,383 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741864_1040 (size=85371)
2024-11-17T12:49:43,384 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/3fc4e4fd62dd4142941b1fd6ee4c44ea, entries=12, sequenceid=198, filesize=17.5 K
2024-11-17T12:49:43,386 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=10.51 KB/10760 for 8f60fdcd6b99d651c379bb67f8bffc8b in 27ms, sequenceid=198, compaction requested=false
2024-11-17T12:49:43,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f60fdcd6b99d651c379bb67f8bffc8b:

2024-11-17T12:49:43,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37879 {}] regionserver.HRegion(8855): Flush requested on 8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:43,387 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f60fdcd6b99d651c379bb67f8bffc8b 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-17T12:49:43,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/d8b0b05f21b141ab802e9745333b8ab3 is 1080, key is row0148/info:/1731847783360/Put/seqid=0
2024-11-17T12:49:43,397 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/f7905615b95148a1b0cc749bba05030b as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/f7905615b95148a1b0cc749bba05030b
2024-11-17T12:49:43,403 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f60fdcd6b99d651c379bb67f8bffc8b/info of 8f60fdcd6b99d651c379bb67f8bffc8b into f7905615b95148a1b0cc749bba05030b(size=83.4 K), total size for store is 100.9 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T12:49:43,403 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f60fdcd6b99d651c379bb67f8bffc8b:

2024-11-17T12:49:43,403 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b., storeName=8f60fdcd6b99d651c379bb67f8bffc8b/info, priority=13, startTime=1731847783358; duration=0sec
2024-11-17T12:49:43,403 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:49:43,403 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f60fdcd6b99d651c379bb67f8bffc8b:info
2024-11-17T12:49:43,408 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741865_1041 (size=16828)
2024-11-17T12:49:43,408 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741865_1041 (size=16828)
2024-11-17T12:49:43,409 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/d8b0b05f21b141ab802e9745333b8ab3
2024-11-17T12:49:43,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/d8b0b05f21b141ab802e9745333b8ab3 as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/d8b0b05f21b141ab802e9745333b8ab3
2024-11-17T12:49:43,420 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/d8b0b05f21b141ab802e9745333b8ab3, entries=11, sequenceid=212, filesize=16.4 K
2024-11-17T12:49:43,421 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=2.10 KB/2152 for 8f60fdcd6b99d651c379bb67f8bffc8b in 34ms, sequenceid=212, compaction requested=true
2024-11-17T12:49:43,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f60fdcd6b99d651c379bb67f8bffc8b:

2024-11-17T12:49:43,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f60fdcd6b99d651c379bb67f8bffc8b:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T12:49:43,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:49:43,421 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T12:49:43,423 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 120105 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-17T12:49:43,423 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1541): 8f60fdcd6b99d651c379bb67f8bffc8b/info is initiating minor compaction (all files)
2024-11-17T12:49:43,423 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f60fdcd6b99d651c379bb67f8bffc8b/info in TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.
2024-11-17T12:49:43,423 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/f7905615b95148a1b0cc749bba05030b, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/3fc4e4fd62dd4142941b1fd6ee4c44ea, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/d8b0b05f21b141ab802e9745333b8ab3] into tmpdir=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp, totalSize=117.3 K
2024-11-17T12:49:43,424 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7905615b95148a1b0cc749bba05030b, keycount=74, bloomtype=ROW, size=83.4 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731847767010
2024-11-17T12:49:43,424 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fc4e4fd62dd4142941b1fd6ee4c44ea, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731847783335
2024-11-17T12:49:43,425 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8b0b05f21b141ab802e9745333b8ab3, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731847783360
2024-11-17T12:49:43,431 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-17T12:49:43,437 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f60fdcd6b99d651c379bb67f8bffc8b#info#compaction#80 average throughput is 49.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T12:49:43,437 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/b5498030ace1480682b222f0d9cbf033 is 1080, key is row0062/info:/1731847767010/Put/seqid=0
2024-11-17T12:49:43,445 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741866_1042 (size=110275)
2024-11-17T12:49:43,445 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741866_1042 (size=110275)
2024-11-17T12:49:43,450 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/b5498030ace1480682b222f0d9cbf033 as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/b5498030ace1480682b222f0d9cbf033
2024-11-17T12:49:43,456 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f60fdcd6b99d651c379bb67f8bffc8b/info of 8f60fdcd6b99d651c379bb67f8bffc8b into b5498030ace1480682b222f0d9cbf033(size=107.7 K), total size for store is 107.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T12:49:43,456 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f60fdcd6b99d651c379bb67f8bffc8b:

2024-11-17T12:49:43,456 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b., storeName=8f60fdcd6b99d651c379bb67f8bffc8b/info, priority=13, startTime=1731847783421; duration=0sec
2024-11-17T12:49:43,457 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:49:43,457 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f60fdcd6b99d651c379bb67f8bffc8b:info
2024-11-17T12:49:43,861 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:43,861 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:44,862 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:44,862 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:45,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37879 {}] regionserver.HRegion(8855): Flush requested on 8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:45,406 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f60fdcd6b99d651c379bb67f8bffc8b 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-17T12:49:45,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/0616f68db5394f69b92deb8830cbac88 is 1080, key is row0159/info:/1731847783388/Put/seqid=0
2024-11-17T12:49:45,415 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741867_1043 (size=12516)
2024-11-17T12:49:45,415 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741867_1043 (size=12516)
2024-11-17T12:49:45,416 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/0616f68db5394f69b92deb8830cbac88
2024-11-17T12:49:45,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/0616f68db5394f69b92deb8830cbac88 as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/0616f68db5394f69b92deb8830cbac88
2024-11-17T12:49:45,428 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/0616f68db5394f69b92deb8830cbac88, entries=7, sequenceid=224, filesize=12.2 K
2024-11-17T12:49:45,428 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for 8f60fdcd6b99d651c379bb67f8bffc8b in 23ms, sequenceid=224, compaction requested=false
2024-11-17T12:49:45,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f60fdcd6b99d651c379bb67f8bffc8b:

2024-11-17T12:49:45,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37879 {}] regionserver.HRegion(8855): Flush requested on 8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:45,430 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f60fdcd6b99d651c379bb67f8bffc8b 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-17T12:49:45,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/5f84bde3f41d4493bf73e27b89753757 is 1080, key is row0166/info:/1731847785407/Put/seqid=0
2024-11-17T12:49:45,441 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741868_1044 (size=17906)
2024-11-17T12:49:45,441 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741868_1044 (size=17906)
2024-11-17T12:49:45,442 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/5f84bde3f41d4493bf73e27b89753757
2024-11-17T12:49:45,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/5f84bde3f41d4493bf73e27b89753757 as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/5f84bde3f41d4493bf73e27b89753757
2024-11-17T12:49:45,454 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/5f84bde3f41d4493bf73e27b89753757, entries=12, sequenceid=239, filesize=17.5 K
2024-11-17T12:49:45,455 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=9.46 KB/9684 for 8f60fdcd6b99d651c379bb67f8bffc8b in 25ms, sequenceid=239, compaction requested=true
2024-11-17T12:49:45,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f60fdcd6b99d651c379bb67f8bffc8b:

2024-11-17T12:49:45,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f60fdcd6b99d651c379bb67f8bffc8b:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T12:49:45,455 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T12:49:45,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:49:45,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37879 {}] regionserver.HRegion(8855): Flush requested on 8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:45,455 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f60fdcd6b99d651c379bb67f8bffc8b 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-11-17T12:49:45,456 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 140697 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-17T12:49:45,456 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1541): 8f60fdcd6b99d651c379bb67f8bffc8b/info is initiating minor compaction (all files)
2024-11-17T12:49:45,456 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f60fdcd6b99d651c379bb67f8bffc8b/info in TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.
2024-11-17T12:49:45,457 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/b5498030ace1480682b222f0d9cbf033, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/0616f68db5394f69b92deb8830cbac88, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/5f84bde3f41d4493bf73e27b89753757] into tmpdir=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp, totalSize=137.4 K
2024-11-17T12:49:45,457 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5498030ace1480682b222f0d9cbf033, keycount=97, bloomtype=ROW, size=107.7 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731847767010
2024-11-17T12:49:45,458 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0616f68db5394f69b92deb8830cbac88, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731847783388
2024-11-17T12:49:45,459 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f84bde3f41d4493bf73e27b89753757, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731847785407
2024-11-17T12:49:45,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/81e19c96614743c6b118ccd88184b21b is 1080, key is row0178/info:/1731847785431/Put/seqid=0
2024-11-17T12:49:45,468 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741869_1045 (size=15750)
2024-11-17T12:49:45,469 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741869_1045 (size=15750)
2024-11-17T12:49:45,469 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/81e19c96614743c6b118ccd88184b21b
2024-11-17T12:49:45,474 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f60fdcd6b99d651c379bb67f8bffc8b#info#compaction#84 average throughput is 39.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T12:49:45,475 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/66e2bf4b8a4944a88aaf63d8e40983e7 is 1080, key is row0062/info:/1731847767010/Put/seqid=0
2024-11-17T12:49:45,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/81e19c96614743c6b118ccd88184b21b as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/81e19c96614743c6b118ccd88184b21b
2024-11-17T12:49:45,480 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741870_1046 (size=130991)
2024-11-17T12:49:45,480 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741870_1046 (size=130991)
2024-11-17T12:49:45,482 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/81e19c96614743c6b118ccd88184b21b, entries=10, sequenceid=252, filesize=15.4 K
2024-11-17T12:49:45,483 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=5.25 KB/5380 for 8f60fdcd6b99d651c379bb67f8bffc8b in 28ms, sequenceid=252, compaction requested=false
2024-11-17T12:49:45,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f60fdcd6b99d651c379bb67f8bffc8b:

2024-11-17T12:49:45,487 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/66e2bf4b8a4944a88aaf63d8e40983e7 as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/66e2bf4b8a4944a88aaf63d8e40983e7
2024-11-17T12:49:45,494 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f60fdcd6b99d651c379bb67f8bffc8b/info of 8f60fdcd6b99d651c379bb67f8bffc8b into 66e2bf4b8a4944a88aaf63d8e40983e7(size=127.9 K), total size for store is 143.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T12:49:45,494 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f60fdcd6b99d651c379bb67f8bffc8b:

2024-11-17T12:49:45,494 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b., storeName=8f60fdcd6b99d651c379bb67f8bffc8b/info, priority=13, startTime=1731847785455; duration=0sec
2024-11-17T12:49:45,494 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:49:45,494 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f60fdcd6b99d651c379bb67f8bffc8b:info
2024-11-17T12:49:45,862 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:45,862 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:46,863 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:46,863 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:47,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37879 {}] regionserver.HRegion(8855): Flush requested on 8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:47,474 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f60fdcd6b99d651c379bb67f8bffc8b 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-17T12:49:47,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/69fd1716868846ad904a8768be12cdca is 1080, key is row0188/info:/1731847785457/Put/seqid=0
2024-11-17T12:49:47,487 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741871_1047 (size=12521)
2024-11-17T12:49:47,488 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741871_1047 (size=12521)
2024-11-17T12:49:47,489 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/69fd1716868846ad904a8768be12cdca
2024-11-17T12:49:47,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/69fd1716868846ad904a8768be12cdca as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/69fd1716868846ad904a8768be12cdca
2024-11-17T12:49:47,501 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/69fd1716868846ad904a8768be12cdca, entries=7, sequenceid=263, filesize=12.2 K
2024-11-17T12:49:47,502 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=10.51 KB/10760 for 8f60fdcd6b99d651c379bb67f8bffc8b in 28ms, sequenceid=263, compaction requested=true
2024-11-17T12:49:47,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f60fdcd6b99d651c379bb67f8bffc8b:

2024-11-17T12:49:47,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f60fdcd6b99d651c379bb67f8bffc8b:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T12:49:47,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:49:47,502 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T12:49:47,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37879 {}] regionserver.HRegion(8855): Flush requested on 8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:47,503 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f60fdcd6b99d651c379bb67f8bffc8b 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-17T12:49:47,503 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 159262 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-17T12:49:47,503 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1541): 8f60fdcd6b99d651c379bb67f8bffc8b/info is initiating minor compaction (all files)
2024-11-17T12:49:47,503 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f60fdcd6b99d651c379bb67f8bffc8b/info in TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.
2024-11-17T12:49:47,503 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/66e2bf4b8a4944a88aaf63d8e40983e7, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/81e19c96614743c6b118ccd88184b21b, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/69fd1716868846ad904a8768be12cdca] into tmpdir=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp, totalSize=155.5 K
2024-11-17T12:49:47,504 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66e2bf4b8a4944a88aaf63d8e40983e7, keycount=116, bloomtype=ROW, size=127.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731847767010
2024-11-17T12:49:47,504 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81e19c96614743c6b118ccd88184b21b, keycount=10, bloomtype=ROW, size=15.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731847785431
2024-11-17T12:49:47,504 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69fd1716868846ad904a8768be12cdca, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731847785457
2024-11-17T12:49:47,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/7dfa9fb2e926422189c2bd25d183581c is 1080, key is row0195/info:/1731847787476/Put/seqid=0
2024-11-17T12:49:47,514 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741872_1048 (size=16839)
2024-11-17T12:49:47,515 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741872_1048 (size=16839)
2024-11-17T12:49:47,519 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f60fdcd6b99d651c379bb67f8bffc8b#info#compaction#87 average throughput is 45.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T12:49:47,519 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/fa559f1f9f1f40cc906e2afff4284978 is 1080, key is row0062/info:/1731847767010/Put/seqid=0
2024-11-17T12:49:47,520 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/7dfa9fb2e926422189c2bd25d183581c
2024-11-17T12:49:47,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/7dfa9fb2e926422189c2bd25d183581c as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/7dfa9fb2e926422189c2bd25d183581c
2024-11-17T12:49:47,533 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/7dfa9fb2e926422189c2bd25d183581c, entries=11, sequenceid=277, filesize=16.4 K
2024-11-17T12:49:47,534 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741873_1049 (size=149497)
2024-11-17T12:49:47,534 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741873_1049 (size=149497)
2024-11-17T12:49:47,534 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=13.66 KB/13988 for 8f60fdcd6b99d651c379bb67f8bffc8b in 31ms, sequenceid=277, compaction requested=false
2024-11-17T12:49:47,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f60fdcd6b99d651c379bb67f8bffc8b:

2024-11-17T12:49:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37879 {}] regionserver.HRegion(8855): Flush requested on 8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:47,536 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f60fdcd6b99d651c379bb67f8bffc8b 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-11-17T12:49:47,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/162de3904bfb42ba93d52a44a16c75f7 is 1080, key is row0206/info:/1731847787504/Put/seqid=0
2024-11-17T12:49:47,549 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741874_1050 (size=20092)
2024-11-17T12:49:47,549 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741874_1050 (size=20092)
2024-11-17T12:49:47,549 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/162de3904bfb42ba93d52a44a16c75f7
2024-11-17T12:49:47,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/162de3904bfb42ba93d52a44a16c75f7 as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/162de3904bfb42ba93d52a44a16c75f7
2024-11-17T12:49:47,560 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/162de3904bfb42ba93d52a44a16c75f7, entries=14, sequenceid=294, filesize=19.6 K
2024-11-17T12:49:47,561 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=5.25 KB/5380 for 8f60fdcd6b99d651c379bb67f8bffc8b in 25ms, sequenceid=294, compaction requested=false
2024-11-17T12:49:47,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f60fdcd6b99d651c379bb67f8bffc8b:

2024-11-17T12:49:47,864 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:47,864 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:47,943 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/fa559f1f9f1f40cc906e2afff4284978 as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/fa559f1f9f1f40cc906e2afff4284978
2024-11-17T12:49:47,951 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f60fdcd6b99d651c379bb67f8bffc8b/info of 8f60fdcd6b99d651c379bb67f8bffc8b into fa559f1f9f1f40cc906e2afff4284978(size=146.0 K), total size for store is 182.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T12:49:47,951 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f60fdcd6b99d651c379bb67f8bffc8b:

2024-11-17T12:49:47,951 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b., storeName=8f60fdcd6b99d651c379bb67f8bffc8b/info, priority=13, startTime=1731847787502; duration=0sec
2024-11-17T12:49:47,951 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:49:47,951 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f60fdcd6b99d651c379bb67f8bffc8b:info
2024-11-17T12:49:48,865 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:48,865 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:49,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37879 {}] regionserver.HRegion(8855): Flush requested on 8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:49,550 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f60fdcd6b99d651c379bb67f8bffc8b 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-17T12:49:49,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/8a923668c156463d917c6ea0704f2650 is 1080, key is row0220/info:/1731847787537/Put/seqid=0
2024-11-17T12:49:49,561 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741875_1051 (size=12523)
2024-11-17T12:49:49,562 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741875_1051 (size=12523)
2024-11-17T12:49:49,562 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/8a923668c156463d917c6ea0704f2650
2024-11-17T12:49:49,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/8a923668c156463d917c6ea0704f2650 as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/8a923668c156463d917c6ea0704f2650
2024-11-17T12:49:49,576 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/8a923668c156463d917c6ea0704f2650, entries=7, sequenceid=305, filesize=12.2 K
2024-11-17T12:49:49,577 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=12.61 KB/12912 for 8f60fdcd6b99d651c379bb67f8bffc8b in 27ms, sequenceid=305, compaction requested=true
2024-11-17T12:49:49,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f60fdcd6b99d651c379bb67f8bffc8b:

2024-11-17T12:49:49,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f60fdcd6b99d651c379bb67f8bffc8b:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T12:49:49,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:49:49,577 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking
2024-11-17T12:49:49,579 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4  files of size 198951 starting at candidate #0 after considering 3 permutations with 3 in ratio
2024-11-17T12:49:49,579 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1541): 8f60fdcd6b99d651c379bb67f8bffc8b/info is initiating minor compaction (all files)
2024-11-17T12:49:49,579 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f60fdcd6b99d651c379bb67f8bffc8b/info in TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.
2024-11-17T12:49:49,579 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/fa559f1f9f1f40cc906e2afff4284978, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/7dfa9fb2e926422189c2bd25d183581c, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/162de3904bfb42ba93d52a44a16c75f7, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/8a923668c156463d917c6ea0704f2650] into tmpdir=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp, totalSize=194.3 K
2024-11-17T12:49:49,579 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa559f1f9f1f40cc906e2afff4284978, keycount=133, bloomtype=ROW, size=146.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731847767010
2024-11-17T12:49:49,580 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7dfa9fb2e926422189c2bd25d183581c, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731847787476
2024-11-17T12:49:49,580 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting 162de3904bfb42ba93d52a44a16c75f7, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1731847787504
2024-11-17T12:49:49,580 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a923668c156463d917c6ea0704f2650, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731847787537
2024-11-17T12:49:49,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37879 {}] regionserver.HRegion(8855): Flush requested on 8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:49,587 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f60fdcd6b99d651c379bb67f8bffc8b 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-17T12:49:49,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/85ae672df6974487bd05c2b8a41a43c4 is 1080, key is row0227/info:/1731847789550/Put/seqid=0
2024-11-17T12:49:49,599 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f60fdcd6b99d651c379bb67f8bffc8b#info#compaction#91 average throughput is 42.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T12:49:49,599 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/ece3df3260b0489e8a22ecb4dcf3ab4b is 1080, key is row0062/info:/1731847767010/Put/seqid=0
2024-11-17T12:49:49,612 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741876_1052 (size=19013)
2024-11-17T12:49:49,612 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741876_1052 (size=19013)
2024-11-17T12:49:49,613 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/85ae672df6974487bd05c2b8a41a43c4
2024-11-17T12:49:49,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/85ae672df6974487bd05c2b8a41a43c4 as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/85ae672df6974487bd05c2b8a41a43c4
2024-11-17T12:49:49,626 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/85ae672df6974487bd05c2b8a41a43c4, entries=13, sequenceid=321, filesize=18.6 K
2024-11-17T12:49:49,627 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=16.81 KB/17216 for 8f60fdcd6b99d651c379bb67f8bffc8b in 40ms, sequenceid=321, compaction requested=false
2024-11-17T12:49:49,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f60fdcd6b99d651c379bb67f8bffc8b:

2024-11-17T12:49:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37879 {}] regionserver.HRegion(8855): Flush requested on 8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:49,627 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f60fdcd6b99d651c379bb67f8bffc8b 1/1 column families, dataSize=17.86 KB heapSize=19.38 KB
2024-11-17T12:49:49,637 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741877_1053 (size=184169)
2024-11-17T12:49:49,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/d98131daa46843c7ad3236f3cc74b908 is 1080, key is row0240/info:/1731847789589/Put/seqid=0
2024-11-17T12:49:49,637 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741877_1053 (size=184169)
2024-11-17T12:49:49,647 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/ece3df3260b0489e8a22ecb4dcf3ab4b as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/ece3df3260b0489e8a22ecb4dcf3ab4b
2024-11-17T12:49:49,652 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741878_1054 (size=23333)
2024-11-17T12:49:49,652 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741878_1054 (size=23333)
2024-11-17T12:49:49,654 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.86 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/d98131daa46843c7ad3236f3cc74b908
2024-11-17T12:49:49,654 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8f60fdcd6b99d651c379bb67f8bffc8b/info of 8f60fdcd6b99d651c379bb67f8bffc8b into ece3df3260b0489e8a22ecb4dcf3ab4b(size=179.9 K), total size for store is 198.4 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T12:49:49,654 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f60fdcd6b99d651c379bb67f8bffc8b:

2024-11-17T12:49:49,654 INFO  [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b., storeName=8f60fdcd6b99d651c379bb67f8bffc8b/info, priority=12, startTime=1731847789577; duration=0sec
2024-11-17T12:49:49,654 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:49:49,654 DEBUG [RS:0;0d751a622496:37879-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f60fdcd6b99d651c379bb67f8bffc8b:info
2024-11-17T12:49:49,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/d98131daa46843c7ad3236f3cc74b908 as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/d98131daa46843c7ad3236f3cc74b908
2024-11-17T12:49:49,710 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/d98131daa46843c7ad3236f3cc74b908, entries=17, sequenceid=341, filesize=22.8 K
2024-11-17T12:49:49,711 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~17.86 KB/18292, heapSize ~19.36 KB/19824, currentSize=0 B/0 for 8f60fdcd6b99d651c379bb67f8bffc8b in 84ms, sequenceid=341, compaction requested=true
2024-11-17T12:49:49,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f60fdcd6b99d651c379bb67f8bffc8b:

2024-11-17T12:49:49,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f60fdcd6b99d651c379bb67f8bffc8b:info, priority=-2147483648, current under compaction store size is 1
2024-11-17T12:49:49,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:49:49,712 DEBUG [RS:0;0d751a622496:37879-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-17T12:49:49,713 DEBUG [RS:0;0d751a622496:37879-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 226515 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-17T12:49:49,713 DEBUG [RS:0;0d751a622496:37879-longCompactions-0 {}] regionserver.HStore(1541): 8f60fdcd6b99d651c379bb67f8bffc8b/info is initiating minor compaction (all files)
2024-11-17T12:49:49,713 INFO  [RS:0;0d751a622496:37879-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f60fdcd6b99d651c379bb67f8bffc8b/info in TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.
2024-11-17T12:49:49,713 INFO  [RS:0;0d751a622496:37879-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/ece3df3260b0489e8a22ecb4dcf3ab4b, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/85ae672df6974487bd05c2b8a41a43c4, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/d98131daa46843c7ad3236f3cc74b908] into tmpdir=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp, totalSize=221.2 K
2024-11-17T12:49:49,714 DEBUG [RS:0;0d751a622496:37879-longCompactions-0 {}] compactions.Compactor(225): Compacting ece3df3260b0489e8a22ecb4dcf3ab4b, keycount=165, bloomtype=ROW, size=179.9 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731847767010
2024-11-17T12:49:49,714 DEBUG [RS:0;0d751a622496:37879-longCompactions-0 {}] compactions.Compactor(225): Compacting 85ae672df6974487bd05c2b8a41a43c4, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731847789550
2024-11-17T12:49:49,714 DEBUG [RS:0;0d751a622496:37879-longCompactions-0 {}] compactions.Compactor(225): Compacting d98131daa46843c7ad3236f3cc74b908, keycount=17, bloomtype=ROW, size=22.8 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731847789589
2024-11-17T12:49:49,729 INFO  [RS:0;0d751a622496:37879-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f60fdcd6b99d651c379bb67f8bffc8b#info#compaction#93 average throughput is 50.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-17T12:49:49,730 DEBUG [RS:0;0d751a622496:37879-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/695564ceef7d458eb3aed6345cfdb58b is 1080, key is row0062/info:/1731847767010/Put/seqid=0
2024-11-17T12:49:49,733 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741879_1055 (size=216718)
2024-11-17T12:49:49,733 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741879_1055 (size=216718)
2024-11-17T12:49:49,742 DEBUG [RS:0;0d751a622496:37879-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/.tmp/info/695564ceef7d458eb3aed6345cfdb58b as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/695564ceef7d458eb3aed6345cfdb58b
2024-11-17T12:49:49,748 INFO  [RS:0;0d751a622496:37879-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f60fdcd6b99d651c379bb67f8bffc8b/info of 8f60fdcd6b99d651c379bb67f8bffc8b into 695564ceef7d458eb3aed6345cfdb58b(size=211.6 K), total size for store is 211.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-17T12:49:49,748 DEBUG [RS:0;0d751a622496:37879-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f60fdcd6b99d651c379bb67f8bffc8b:

2024-11-17T12:49:49,748 INFO  [RS:0;0d751a622496:37879-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b., storeName=8f60fdcd6b99d651c379bb67f8bffc8b/info, priority=13, startTime=1731847789711; duration=0sec
2024-11-17T12:49:49,748 DEBUG [RS:0;0d751a622496:37879-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-17T12:49:49,748 DEBUG [RS:0;0d751a622496:37879-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f60fdcd6b99d651c379bb67f8bffc8b:info
2024-11-17T12:49:49,865 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:49,865 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:50,866 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:50,866 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:51,629 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files
2024-11-17T12:49:51,629 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C37879%2C1731847753619.1731847791629
2024-11-17T12:49:51,634 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:51,634 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:51,634 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:51,634 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:51,634 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:51,635 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/WALs/0d751a622496,37879,1731847753619/0d751a622496%2C37879%2C1731847753619.1731847754189 with entries=320, filesize=311.04 KB; new WAL /user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/WALs/0d751a622496,37879,1731847753619/0d751a622496%2C37879%2C1731847753619.1731847791629
2024-11-17T12:49:51,636 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741833_1009 (size=318509)
2024-11-17T12:49:51,636 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741833_1009 (size=318509)
2024-11-17T12:49:51,637 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35291:35291),(127.0.0.1/127.0.0.1:32955:32955)]
2024-11-17T12:49:51,637 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/WALs/0d751a622496,37879,1731847753619/0d751a622496%2C37879%2C1731847753619.1731847754189 is not closed yet, will try archiving it next time
2024-11-17T12:49:51,637 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/WALs/0d751a622496,37879,1731847753619/0d751a622496%2C37879%2C1731847753619.1731847754189 to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/oldWALs/0d751a622496%2C37879%2C1731847753619.1731847754189
2024-11-17T12:49:51,641 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8f60fdcd6b99d651c379bb67f8bffc8b:

2024-11-17T12:49:51,642 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e36f819d8f0d43e1647618946ce7d017:

2024-11-17T12:49:51,642 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=705 B heapSize=2.05 KB
2024-11-17T12:49:51,646 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/hbase/meta/1588230740/.tmp/info/36c86cee858d448c89426ebfc54b4d02 is 193, key is TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b./info:regioninfo/1731847769859/Put/seqid=0
2024-11-17T12:49:51,650 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741881_1057 (size=6223)
2024-11-17T12:49:51,650 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741881_1057 (size=6223)
2024-11-17T12:49:51,650 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=705 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/hbase/meta/1588230740/.tmp/info/36c86cee858d448c89426ebfc54b4d02
2024-11-17T12:49:51,657 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/hbase/meta/1588230740/.tmp/info/36c86cee858d448c89426ebfc54b4d02 as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/hbase/meta/1588230740/info/36c86cee858d448c89426ebfc54b4d02
2024-11-17T12:49:51,662 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/hbase/meta/1588230740/info/36c86cee858d448c89426ebfc54b4d02, entries=5, sequenceid=21, filesize=6.1 K
2024-11-17T12:49:51,662 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~705 B/705, heapSize ~1.29 KB/1320, currentSize=0 B/0 for 1588230740 in 20ms, sequenceid=21, compaction requested=false
2024-11-17T12:49:51,663 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-17T12:49:51,663 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1])
2024-11-17T12:49:51,663 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-17T12:49:51,663 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T12:49:51,664 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T12:49:51,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:49:51,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:49:51,664 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-17T12:49:51,664 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-17T12:49:51,664 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1217753638, stopped=false
2024-11-17T12:49:51,664 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0d751a622496,42271,1731847753456
2024-11-17T12:49:51,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T12:49:51,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:51,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37879-0x1014a21219e0001, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T12:49:51,730 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T12:49:51,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37879-0x1014a21219e0001, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:51,730 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T12:49:51,730 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T12:49:51,730 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:49:51,731 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0d751a622496,37879,1731847753619' *****
2024-11-17T12:49:51,731 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-17T12:49:51,731 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37879-0x1014a21219e0001, quorum=127.0.0.1:61606, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:49:51,731 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:49:51,731 INFO  [RS:0;0d751a622496:37879 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-17T12:49:51,731 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-17T12:49:51,731 INFO  [RS:0;0d751a622496:37879 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-17T12:49:51,731 INFO  [RS:0;0d751a622496:37879 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-17T12:49:51,732 INFO  [RS:0;0d751a622496:37879 {}] regionserver.HRegionServer(3091): Received CLOSE for 8f60fdcd6b99d651c379bb67f8bffc8b
2024-11-17T12:49:51,732 INFO  [RS:0;0d751a622496:37879 {}] regionserver.HRegionServer(3091): Received CLOSE for e36f819d8f0d43e1647618946ce7d017
2024-11-17T12:49:51,732 INFO  [RS:0;0d751a622496:37879 {}] regionserver.HRegionServer(959): stopping server 0d751a622496,37879,1731847753619
2024-11-17T12:49:51,732 INFO  [RS:0;0d751a622496:37879 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T12:49:51,732 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8f60fdcd6b99d651c379bb67f8bffc8b, disabling compactions & flushes
2024-11-17T12:49:51,732 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.
2024-11-17T12:49:51,732 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.
2024-11-17T12:49:51,732 INFO  [RS:0;0d751a622496:37879 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0d751a622496:37879.
2024-11-17T12:49:51,732 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b. after waiting 0 ms
2024-11-17T12:49:51,732 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.
2024-11-17T12:49:51,732 DEBUG [RS:0;0d751a622496:37879 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T12:49:51,732 DEBUG [RS:0;0d751a622496:37879 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:49:51,732 INFO  [RS:0;0d751a622496:37879 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-17T12:49:51,732 INFO  [RS:0;0d751a622496:37879 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-17T12:49:51,732 INFO  [RS:0;0d751a622496:37879 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-17T12:49:51,732 INFO  [RS:0;0d751a622496:37879 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-17T12:49:51,732 INFO  [RS:0;0d751a622496:37879 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close
2024-11-17T12:49:51,732 DEBUG [RS:0;0d751a622496:37879 {}] regionserver.HRegionServer(1325): Online Regions={8f60fdcd6b99d651c379bb67f8bffc8b=TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b., e36f819d8f0d43e1647618946ce7d017=TestLogRolling-testLogRolling,,1731847769116.e36f819d8f0d43e1647618946ce7d017., 1588230740=hbase:meta,,1.1588230740}
2024-11-17T12:49:51,732 DEBUG [RS:0;0d751a622496:37879 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 8f60fdcd6b99d651c379bb67f8bffc8b, e36f819d8f0d43e1647618946ce7d017
2024-11-17T12:49:51,732 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T12:49:51,733 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T12:49:51,733 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T12:49:51,733 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T12:49:51,733 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T12:49:51,732 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/be7b1ecc88d647b18ec40ad9f2004c7f.d6b23169599b74b84dd92cc0cf93118e->hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/be7b1ecc88d647b18ec40ad9f2004c7f-top, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/TestLogRolling-testLogRolling=d6b23169599b74b84dd92cc0cf93118e-3624f13c2494474fa7b8747dcff910ec, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/14c2b5c8ed0a4b73964a6e08b8a4fd7d, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/TestLogRolling-testLogRolling=d6b23169599b74b84dd92cc0cf93118e-67427ed6f2ae40e99e14ff9908858a0c, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/0e24fdbe21d1492faea2a4dc40c6382b, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/8890a648fe534f77a2bbb6d150235bbe, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/9f6e33cb33854f0ca5fb5d739d0e928c, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/ed1d2fe5e9d74c47954103cce804338d, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/f7905615b95148a1b0cc749bba05030b, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/10bdead0bdce4550a644275e6fff4c06, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/3fc4e4fd62dd4142941b1fd6ee4c44ea, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/b5498030ace1480682b222f0d9cbf033, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/d8b0b05f21b141ab802e9745333b8ab3, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/0616f68db5394f69b92deb8830cbac88, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/66e2bf4b8a4944a88aaf63d8e40983e7, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/5f84bde3f41d4493bf73e27b89753757, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/81e19c96614743c6b118ccd88184b21b, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/fa559f1f9f1f40cc906e2afff4284978, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/69fd1716868846ad904a8768be12cdca, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/7dfa9fb2e926422189c2bd25d183581c, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/162de3904bfb42ba93d52a44a16c75f7, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/ece3df3260b0489e8a22ecb4dcf3ab4b, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/8a923668c156463d917c6ea0704f2650, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/85ae672df6974487bd05c2b8a41a43c4, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/d98131daa46843c7ad3236f3cc74b908] to archive
2024-11-17T12:49:51,734 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-17T12:49:51,736 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/be7b1ecc88d647b18ec40ad9f2004c7f.d6b23169599b74b84dd92cc0cf93118e to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/be7b1ecc88d647b18ec40ad9f2004c7f.d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:51,737 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/TestLogRolling-testLogRolling=d6b23169599b74b84dd92cc0cf93118e-3624f13c2494474fa7b8747dcff910ec to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/TestLogRolling-testLogRolling=d6b23169599b74b84dd92cc0cf93118e-3624f13c2494474fa7b8747dcff910ec
2024-11-17T12:49:51,738 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/14c2b5c8ed0a4b73964a6e08b8a4fd7d to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/14c2b5c8ed0a4b73964a6e08b8a4fd7d
2024-11-17T12:49:51,740 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/TestLogRolling-testLogRolling=d6b23169599b74b84dd92cc0cf93118e-67427ed6f2ae40e99e14ff9908858a0c to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/TestLogRolling-testLogRolling=d6b23169599b74b84dd92cc0cf93118e-67427ed6f2ae40e99e14ff9908858a0c
2024-11-17T12:49:51,741 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/0e24fdbe21d1492faea2a4dc40c6382b to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/0e24fdbe21d1492faea2a4dc40c6382b
2024-11-17T12:49:51,741 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1
2024-11-17T12:49:51,742 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/8890a648fe534f77a2bbb6d150235bbe to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/8890a648fe534f77a2bbb6d150235bbe
2024-11-17T12:49:51,742 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T12:49:51,742 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T12:49:51,742 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731847791732Running coprocessor pre-close hooks at 1731847791732Disabling compacts and flushes for region at 1731847791732Disabling writes for close at 1731847791733 (+1 ms)Writing region close event to WAL at 1731847791739 (+6 ms)Running coprocessor post-close hooks at 1731847791742 (+3 ms)Closed at 1731847791742
2024-11-17T12:49:51,742 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-17T12:49:51,743 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/9f6e33cb33854f0ca5fb5d739d0e928c to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/9f6e33cb33854f0ca5fb5d739d0e928c
2024-11-17T12:49:51,744 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/ed1d2fe5e9d74c47954103cce804338d to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/ed1d2fe5e9d74c47954103cce804338d
2024-11-17T12:49:51,745 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/f7905615b95148a1b0cc749bba05030b to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/f7905615b95148a1b0cc749bba05030b
2024-11-17T12:49:51,746 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/10bdead0bdce4550a644275e6fff4c06 to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/10bdead0bdce4550a644275e6fff4c06
2024-11-17T12:49:51,747 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/3fc4e4fd62dd4142941b1fd6ee4c44ea to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/3fc4e4fd62dd4142941b1fd6ee4c44ea
2024-11-17T12:49:51,749 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/b5498030ace1480682b222f0d9cbf033 to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/b5498030ace1480682b222f0d9cbf033
2024-11-17T12:49:51,750 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/d8b0b05f21b141ab802e9745333b8ab3 to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/d8b0b05f21b141ab802e9745333b8ab3
2024-11-17T12:49:51,751 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/0616f68db5394f69b92deb8830cbac88 to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/0616f68db5394f69b92deb8830cbac88
2024-11-17T12:49:51,752 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/66e2bf4b8a4944a88aaf63d8e40983e7 to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/66e2bf4b8a4944a88aaf63d8e40983e7
2024-11-17T12:49:51,753 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/5f84bde3f41d4493bf73e27b89753757 to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/5f84bde3f41d4493bf73e27b89753757
2024-11-17T12:49:51,754 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/81e19c96614743c6b118ccd88184b21b to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/81e19c96614743c6b118ccd88184b21b
2024-11-17T12:49:51,755 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/fa559f1f9f1f40cc906e2afff4284978 to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/fa559f1f9f1f40cc906e2afff4284978
2024-11-17T12:49:51,756 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/69fd1716868846ad904a8768be12cdca to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/69fd1716868846ad904a8768be12cdca
2024-11-17T12:49:51,757 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/7dfa9fb2e926422189c2bd25d183581c to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/7dfa9fb2e926422189c2bd25d183581c
2024-11-17T12:49:51,758 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/162de3904bfb42ba93d52a44a16c75f7 to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/162de3904bfb42ba93d52a44a16c75f7
2024-11-17T12:49:51,759 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/ece3df3260b0489e8a22ecb4dcf3ab4b to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/ece3df3260b0489e8a22ecb4dcf3ab4b
2024-11-17T12:49:51,761 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/8a923668c156463d917c6ea0704f2650 to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/8a923668c156463d917c6ea0704f2650
2024-11-17T12:49:51,762 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/85ae672df6974487bd05c2b8a41a43c4 to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/85ae672df6974487bd05c2b8a41a43c4
2024-11-17T12:49:51,763 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/d98131daa46843c7ad3236f3cc74b908 to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/info/d98131daa46843c7ad3236f3cc74b908
2024-11-17T12:49:51,763 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=0d751a622496:42271 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	... 16 more
2024-11-17T12:49:51,763 WARN  [StoreCloser-TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [14c2b5c8ed0a4b73964a6e08b8a4fd7d=42984, 0e24fdbe21d1492faea2a4dc40c6382b=12516, 8890a648fe534f77a2bbb6d150235bbe=63636, 9f6e33cb33854f0ca5fb5d739d0e928c=17906, ed1d2fe5e9d74c47954103cce804338d=19000, f7905615b95148a1b0cc749bba05030b=85371, 10bdead0bdce4550a644275e6fff4c06=12516, 3fc4e4fd62dd4142941b1fd6ee4c44ea=17906, b5498030ace1480682b222f0d9cbf033=110275, d8b0b05f21b141ab802e9745333b8ab3=16828, 0616f68db5394f69b92deb8830cbac88=12516, 66e2bf4b8a4944a88aaf63d8e40983e7=130991, 5f84bde3f41d4493bf73e27b89753757=17906, 81e19c96614743c6b118ccd88184b21b=15750, fa559f1f9f1f40cc906e2afff4284978=149497, 69fd1716868846ad904a8768be12cdca=12521, 7dfa9fb2e926422189c2bd25d183581c=16839, 162de3904bfb42ba93d52a44a16c75f7=20092, ece3df3260b0489e8a22ecb4dcf3ab4b=184169, 8a923668c156463d917c6ea0704f2650=12523, 85ae672df6974487bd05c2b8a41a43c4=19013, d98131daa46843c7ad3236f3cc74b908=23333]
2024-11-17T12:49:51,767 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/8f60fdcd6b99d651c379bb67f8bffc8b/recovered.edits/347.seqid, newMaxSeqId=347, maxSeqId=130
2024-11-17T12:49:51,767 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.
2024-11-17T12:49:51,767 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8f60fdcd6b99d651c379bb67f8bffc8b:
Waiting for close lock at 1731847791732Running coprocessor pre-close hooks at 1731847791732Disabling compacts and flushes for region at 1731847791732Disabling writes for close at 1731847791732Writing region close event to WAL at 1731847791764 (+32 ms)Running coprocessor post-close hooks at 1731847791767 (+3 ms)Closed at 1731847791767
2024-11-17T12:49:51,767 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1731847769116.8f60fdcd6b99d651c379bb67f8bffc8b.
2024-11-17T12:49:51,768 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e36f819d8f0d43e1647618946ce7d017, disabling compactions & flushes
2024-11-17T12:49:51,768 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731847769116.e36f819d8f0d43e1647618946ce7d017.
2024-11-17T12:49:51,768 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731847769116.e36f819d8f0d43e1647618946ce7d017.
2024-11-17T12:49:51,768 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731847769116.e36f819d8f0d43e1647618946ce7d017. after waiting 0 ms
2024-11-17T12:49:51,768 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731847769116.e36f819d8f0d43e1647618946ce7d017.
2024-11-17T12:49:51,768 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731847769116.e36f819d8f0d43e1647618946ce7d017.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/e36f819d8f0d43e1647618946ce7d017/info/be7b1ecc88d647b18ec40ad9f2004c7f.d6b23169599b74b84dd92cc0cf93118e->hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/d6b23169599b74b84dd92cc0cf93118e/info/be7b1ecc88d647b18ec40ad9f2004c7f-bottom] to archive
2024-11-17T12:49:51,769 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731847769116.e36f819d8f0d43e1647618946ce7d017.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-17T12:49:51,770 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731847769116.e36f819d8f0d43e1647618946ce7d017.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/e36f819d8f0d43e1647618946ce7d017/info/be7b1ecc88d647b18ec40ad9f2004c7f.d6b23169599b74b84dd92cc0cf93118e to hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/archive/data/default/TestLogRolling-testLogRolling/e36f819d8f0d43e1647618946ce7d017/info/be7b1ecc88d647b18ec40ad9f2004c7f.d6b23169599b74b84dd92cc0cf93118e
2024-11-17T12:49:51,771 WARN  [StoreCloser-TestLogRolling-testLogRolling,,1731847769116.e36f819d8f0d43e1647618946ce7d017.-1 {}] regionserver.HStore(2414): Failed to report archival of files: []
2024-11-17T12:49:51,775 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/data/default/TestLogRolling-testLogRolling/e36f819d8f0d43e1647618946ce7d017/recovered.edits/135.seqid, newMaxSeqId=135, maxSeqId=130
2024-11-17T12:49:51,776 INFO  [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731847769116.e36f819d8f0d43e1647618946ce7d017.
2024-11-17T12:49:51,776 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e36f819d8f0d43e1647618946ce7d017:
Waiting for close lock at 1731847791768Running coprocessor pre-close hooks at 1731847791768Disabling compacts and flushes for region at 1731847791768Disabling writes for close at 1731847791768Writing region close event to WAL at 1731847791772 (+4 ms)Running coprocessor post-close hooks at 1731847791775 (+3 ms)Closed at 1731847791775
2024-11-17T12:49:51,776 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1731847769116.e36f819d8f0d43e1647618946ce7d017.
2024-11-17T12:49:51,867 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:51,867 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:51,933 INFO  [RS:0;0d751a622496:37879 {}] regionserver.HRegionServer(976): stopping server 0d751a622496,37879,1731847753619; all regions closed.
2024-11-17T12:49:51,933 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:51,933 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:51,933 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:51,934 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:51,934 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:51,935 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741834_1010 (size=8107)
2024-11-17T12:49:51,936 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741834_1010 (size=8107)
2024-11-17T12:49:51,937 DEBUG [RS:0;0d751a622496:37879 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/oldWALs
2024-11-17T12:49:51,937 INFO  [RS:0;0d751a622496:37879 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0d751a622496%2C37879%2C1731847753619.meta:.meta(num 1731847754631)
2024-11-17T12:49:51,938 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:51,938 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:51,938 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:51,938 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:51,938 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:51,940 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741880_1056 (size=780)
2024-11-17T12:49:51,940 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741880_1056 (size=780)
2024-11-17T12:49:51,942 DEBUG [RS:0;0d751a622496:37879 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/oldWALs
2024-11-17T12:49:51,943 INFO  [RS:0;0d751a622496:37879 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0d751a622496%2C37879%2C1731847753619:(num 1731847791629)
2024-11-17T12:49:51,943 DEBUG [RS:0;0d751a622496:37879 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:49:51,943 INFO  [RS:0;0d751a622496:37879 {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T12:49:51,943 INFO  [RS:0;0d751a622496:37879 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T12:49:51,943 INFO  [RS:0;0d751a622496:37879 {}] hbase.ChoreService(370): Chore service for: regionserver/0d751a622496:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-17T12:49:51,943 INFO  [RS:0;0d751a622496:37879 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T12:49:51,943 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T12:49:51,943 INFO  [RS:0;0d751a622496:37879 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37879
2024-11-17T12:49:51,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37879-0x1014a21219e0001, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0d751a622496,37879,1731847753619
2024-11-17T12:49:51,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T12:49:51,971 INFO  [RS:0;0d751a622496:37879 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T12:49:51,979 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0d751a622496,37879,1731847753619]
2024-11-17T12:49:51,988 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0d751a622496,37879,1731847753619 already deleted, retry=false
2024-11-17T12:49:51,988 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0d751a622496,37879,1731847753619 expired; onlineServers=0
2024-11-17T12:49:51,988 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0d751a622496,42271,1731847753456' *****
2024-11-17T12:49:51,988 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-17T12:49:51,988 INFO  [M:0;0d751a622496:42271 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T12:49:51,988 INFO  [M:0;0d751a622496:42271 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T12:49:51,988 DEBUG [M:0;0d751a622496:42271 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-17T12:49:51,988 DEBUG [M:0;0d751a622496:42271 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-17T12:49:51,988 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-17T12:49:51,988 DEBUG [master/0d751a622496:0:becomeActiveMaster-HFileCleaner.large.0-1731847753950 {}] cleaner.HFileCleaner(306): Exit Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.large.0-1731847753950,5,FailOnTimeoutGroup]
2024-11-17T12:49:51,988 DEBUG [master/0d751a622496:0:becomeActiveMaster-HFileCleaner.small.0-1731847753951 {}] cleaner.HFileCleaner(306): Exit Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.small.0-1731847753951,5,FailOnTimeoutGroup]
2024-11-17T12:49:51,988 INFO  [M:0;0d751a622496:42271 {}] hbase.ChoreService(370): Chore service for: master/0d751a622496:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-17T12:49:51,988 INFO  [M:0;0d751a622496:42271 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T12:49:51,989 DEBUG [M:0;0d751a622496:42271 {}] master.HMaster(1795): Stopping service threads
2024-11-17T12:49:51,989 INFO  [M:0;0d751a622496:42271 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-17T12:49:51,989 INFO  [M:0;0d751a622496:42271 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T12:49:51,989 INFO  [M:0;0d751a622496:42271 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-17T12:49:51,989 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-17T12:49:51,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-17T12:49:51,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:51,996 DEBUG [M:0;0d751a622496:42271 {}] zookeeper.ZKUtil(347): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-17T12:49:51,996 WARN  [M:0;0d751a622496:42271 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-17T12:49:51,997 INFO  [M:0;0d751a622496:42271 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/.lastflushedseqids
2024-11-17T12:49:52,003 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741882_1058 (size=228)
2024-11-17T12:49:52,003 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741882_1058 (size=228)
2024-11-17T12:49:52,004 INFO  [M:0;0d751a622496:42271 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-17T12:49:52,004 INFO  [M:0;0d751a622496:42271 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-17T12:49:52,004 DEBUG [M:0;0d751a622496:42271 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T12:49:52,004 INFO  [M:0;0d751a622496:42271 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:49:52,004 DEBUG [M:0;0d751a622496:42271 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:49:52,004 DEBUG [M:0;0d751a622496:42271 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T12:49:52,004 DEBUG [M:0;0d751a622496:42271 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:49:52,004 INFO  [M:0;0d751a622496:42271 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.43 KB heapSize=63.38 KB
2024-11-17T12:49:52,022 DEBUG [M:0;0d751a622496:42271 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1a1ab56a6523424693e4ec3ca5f3e9fe is 82, key is hbase:meta,,1/info:regioninfo/1731847754657/Put/seqid=0
2024-11-17T12:49:52,026 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741883_1059 (size=5672)
2024-11-17T12:49:52,026 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741883_1059 (size=5672)
2024-11-17T12:49:52,026 INFO  [M:0;0d751a622496:42271 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1a1ab56a6523424693e4ec3ca5f3e9fe
2024-11-17T12:49:52,045 DEBUG [M:0;0d751a622496:42271 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6ca1837046634adc98df7f159d766f76 is 750, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731847755150/Put/seqid=0
2024-11-17T12:49:52,048 INFO  [regionserver/0d751a622496:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T12:49:52,048 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741884_1060 (size=7090)
2024-11-17T12:49:52,049 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741884_1060 (size=7090)
2024-11-17T12:49:52,049 INFO  [M:0;0d751a622496:42271 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.83 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6ca1837046634adc98df7f159d766f76
2024-11-17T12:49:52,053 INFO  [M:0;0d751a622496:42271 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6ca1837046634adc98df7f159d766f76
2024-11-17T12:49:52,067 DEBUG [M:0;0d751a622496:42271 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/369fb31a985541f9b63a000110a036d1 is 69, key is 0d751a622496,37879,1731847753619/rs:state/1731847753995/Put/seqid=0
2024-11-17T12:49:52,071 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741885_1061 (size=5156)
2024-11-17T12:49:52,072 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741885_1061 (size=5156)
2024-11-17T12:49:52,072 INFO  [M:0;0d751a622496:42271 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/369fb31a985541f9b63a000110a036d1
2024-11-17T12:49:52,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37879-0x1014a21219e0001, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:49:52,080 INFO  [RS:0;0d751a622496:37879 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T12:49:52,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37879-0x1014a21219e0001, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:49:52,080 INFO  [RS:0;0d751a622496:37879 {}] regionserver.HRegionServer(1031): Exiting; stopping=0d751a622496,37879,1731847753619; zookeeper connection closed.
2024-11-17T12:49:52,080 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2d491842 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2d491842
2024-11-17T12:49:52,080 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-17T12:49:52,089 DEBUG [M:0;0d751a622496:42271 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/66d1cef52b424985823fcc02f072e296 is 52, key is load_balancer_on/state:d/1731847754768/Put/seqid=0
2024-11-17T12:49:52,093 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741886_1062 (size=5056)
2024-11-17T12:49:52,093 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741886_1062 (size=5056)
2024-11-17T12:49:52,094 INFO  [M:0;0d751a622496:42271 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/66d1cef52b424985823fcc02f072e296
2024-11-17T12:49:52,098 DEBUG [M:0;0d751a622496:42271 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1a1ab56a6523424693e4ec3ca5f3e9fe as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1a1ab56a6523424693e4ec3ca5f3e9fe
2024-11-17T12:49:52,101 INFO  [M:0;0d751a622496:42271 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1a1ab56a6523424693e4ec3ca5f3e9fe, entries=8, sequenceid=125, filesize=5.5 K
2024-11-17T12:49:52,102 DEBUG [M:0;0d751a622496:42271 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6ca1837046634adc98df7f159d766f76 as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6ca1837046634adc98df7f159d766f76
2024-11-17T12:49:52,106 INFO  [M:0;0d751a622496:42271 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6ca1837046634adc98df7f159d766f76
2024-11-17T12:49:52,106 INFO  [M:0;0d751a622496:42271 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6ca1837046634adc98df7f159d766f76, entries=13, sequenceid=125, filesize=6.9 K
2024-11-17T12:49:52,107 DEBUG [M:0;0d751a622496:42271 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/369fb31a985541f9b63a000110a036d1 as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/369fb31a985541f9b63a000110a036d1
2024-11-17T12:49:52,111 INFO  [M:0;0d751a622496:42271 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/369fb31a985541f9b63a000110a036d1, entries=1, sequenceid=125, filesize=5.0 K
2024-11-17T12:49:52,112 DEBUG [M:0;0d751a622496:42271 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/66d1cef52b424985823fcc02f072e296 as hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/66d1cef52b424985823fcc02f072e296
2024-11-17T12:49:52,116 INFO  [M:0;0d751a622496:42271 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41193/user/jenkins/test-data/d4cc4a91-c611-8926-941a-0c47201280ae/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/66d1cef52b424985823fcc02f072e296, entries=1, sequenceid=125, filesize=4.9 K
2024-11-17T12:49:52,117 INFO  [M:0;0d751a622496:42271 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.43 KB/52663, heapSize ~63.32 KB/64840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 113ms, sequenceid=125, compaction requested=false
2024-11-17T12:49:52,119 INFO  [M:0;0d751a622496:42271 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:49:52,119 DEBUG [M:0;0d751a622496:42271 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731847792004Disabling compacts and flushes for region at 1731847792004Disabling writes for close at 1731847792004Obtaining lock to block concurrent updates at 1731847792004Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731847792004Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52663, getHeapSize=64840, getOffHeapSize=0, getCellsCount=148 at 1731847792005 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731847792006 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731847792006Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731847792021 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731847792021Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731847792030 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731847792044 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731847792044Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731847792053 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731847792067 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731847792067Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731847792076 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731847792088 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731847792088Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a024d75: reopening flushed file at 1731847792097 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32170cb: reopening flushed file at 1731847792102 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17e2592e: reopening flushed file at 1731847792106 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a4a767c: reopening flushed file at 1731847792111 (+5 ms)Finished flush of dataSize ~51.43 KB/52663, heapSize ~63.32 KB/64840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 113ms, sequenceid=125, compaction requested=false at 1731847792117 (+6 ms)Writing region close event to WAL at 1731847792119 (+2 ms)Closed at 1731847792119
2024-11-17T12:49:52,119 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:52,119 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:52,119 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:52,119 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:52,119 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:52,121 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39689 is added to blk_1073741830_1006 (size=61332)
2024-11-17T12:49:52,121 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35075 is added to blk_1073741830_1006 (size=61332)
2024-11-17T12:49:52,122 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T12:49:52,122 INFO  [M:0;0d751a622496:42271 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-17T12:49:52,122 INFO  [M:0;0d751a622496:42271 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42271
2024-11-17T12:49:52,122 INFO  [M:0;0d751a622496:42271 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T12:49:52,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:49:52,251 INFO  [M:0;0d751a622496:42271 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T12:49:52,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42271-0x1014a21219e0000, quorum=127.0.0.1:61606, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:49:52,285 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4b81e396{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:49:52,285 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2058e2af{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:49:52,285 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:49:52,285 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7cbc9241{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:49:52,286 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5bbaae64{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/hadoop.log.dir/,STOPPED}
2024-11-17T12:49:52,288 WARN  [BP-903435966-172.17.0.2-1731847751586 heartbeating to localhost/127.0.0.1:41193 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T12:49:52,288 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T12:49:52,288 WARN  [BP-903435966-172.17.0.2-1731847751586 heartbeating to localhost/127.0.0.1:41193 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-903435966-172.17.0.2-1731847751586 (Datanode Uuid 1fa05234-3f75-49b6-b36d-70ec633312c1) service to localhost/127.0.0.1:41193
2024-11-17T12:49:52,288 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T12:49:52,288 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/cluster_c97ab916-9cf5-6219-161a-81127106f55c/data/data3/current/BP-903435966-172.17.0.2-1731847751586 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:49:52,288 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/cluster_c97ab916-9cf5-6219-161a-81127106f55c/data/data4/current/BP-903435966-172.17.0.2-1731847751586 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:49:52,289 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T12:49:52,291 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@444decb3{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:49:52,291 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@216b0c63{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:49:52,291 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:49:52,291 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ab86f9f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:49:52,291 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b4ed6ad{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/hadoop.log.dir/,STOPPED}
2024-11-17T12:49:52,293 WARN  [BP-903435966-172.17.0.2-1731847751586 heartbeating to localhost/127.0.0.1:41193 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T12:49:52,293 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T12:49:52,293 WARN  [BP-903435966-172.17.0.2-1731847751586 heartbeating to localhost/127.0.0.1:41193 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-903435966-172.17.0.2-1731847751586 (Datanode Uuid dd44247c-6fba-49b4-a979-dafe76384873) service to localhost/127.0.0.1:41193
2024-11-17T12:49:52,293 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T12:49:52,293 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/cluster_c97ab916-9cf5-6219-161a-81127106f55c/data/data1/current/BP-903435966-172.17.0.2-1731847751586 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:49:52,293 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/cluster_c97ab916-9cf5-6219-161a-81127106f55c/data/data2/current/BP-903435966-172.17.0.2-1731847751586 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:49:52,294 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T12:49:52,298 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6a6db152{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T12:49:52,299 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@79aa061{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:49:52,299 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:49:52,299 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@47cbf00e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:49:52,299 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@78512cf7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/hadoop.log.dir/,STOPPED}
2024-11-17T12:49:52,306 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-17T12:49:52,336 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-17T12:49:52,343 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=229 (was 204)
Potentially hanging thread: nioEventLoopGroup-39-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41193
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41193
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41193 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-39-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:41193
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41193 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-38-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:41193
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41193 from jenkins.hfs.6
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-40-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41193
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=515 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=219 (was 220), ProcessCount=11 (was 11), AvailableMemoryMB=4998 (was 5159)
2024-11-17T12:49:52,351 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=229, OpenFileDescriptor=515, MaxFileDescriptor=1048576, SystemLoadAverage=219, ProcessCount=11, AvailableMemoryMB=4998
2024-11-17T12:49:52,351 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-17T12:49:52,351 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/hadoop.log.dir so I do NOT create it in target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c
2024-11-17T12:49:52,351 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7928deaa-c68d-e0b5-af69-fef2a3354b29/hadoop.tmp.dir so I do NOT create it in target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c
2024-11-17T12:49:52,351 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/cluster_f4408350-8688-a6d1-a2d2-82f46e2d86c0, deleteOnExit=true
2024-11-17T12:49:52,351 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-17T12:49:52,352 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/test.cache.data in system properties and HBase conf
2024-11-17T12:49:52,352 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/hadoop.tmp.dir in system properties and HBase conf
2024-11-17T12:49:52,352 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/hadoop.log.dir in system properties and HBase conf
2024-11-17T12:49:52,352 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-17T12:49:52,352 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-17T12:49:52,352 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-17T12:49:52,352 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-17T12:49:52,352 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-17T12:49:52,352 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-17T12:49:52,352 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-17T12:49:52,352 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T12:49:52,353 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-17T12:49:52,353 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-17T12:49:52,353 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-17T12:49:52,353 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T12:49:52,353 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-17T12:49:52,353 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/nfs.dump.dir in system properties and HBase conf
2024-11-17T12:49:52,353 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/java.io.tmpdir in system properties and HBase conf
2024-11-17T12:49:52,353 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-17T12:49:52,353 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-17T12:49:52,353 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-17T12:49:52,366 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T12:49:52,624 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:49:52,627 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:49:52,628 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:49:52,628 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:49:52,628 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-17T12:49:52,628 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:49:52,629 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7d664f93{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:49:52,629 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3c703483{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:49:52,720 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6148e76b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/java.io.tmpdir/jetty-localhost-42149-hadoop-hdfs-3_4_1-tests_jar-_-any-17975307142935059749/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T12:49:52,721 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3b55cb8{HTTP/1.1, (http/1.1)}{localhost:42149}
2024-11-17T12:49:52,721 INFO  [Time-limited test {}] server.Server(415): Started @287963ms
2024-11-17T12:49:52,732 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-17T12:49:52,868 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:52,868 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:52,910 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:49:52,913 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:49:52,914 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:49:52,914 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:49:52,914 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-17T12:49:52,914 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a1af98b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:49:52,915 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4bb82bb7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:49:53,012 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@23e03366{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/java.io.tmpdir/jetty-localhost-43409-hadoop-hdfs-3_4_1-tests_jar-_-any-12597075280125498543/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:49:53,012 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2fb7753b{HTTP/1.1, (http/1.1)}{localhost:43409}
2024-11-17T12:49:53,012 INFO  [Time-limited test {}] server.Server(415): Started @288254ms
2024-11-17T12:49:53,013 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T12:49:53,037 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-17T12:49:53,039 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-17T12:49:53,040 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-17T12:49:53,040 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-17T12:49:53,040 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-17T12:49:53,040 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@22ed154c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/hadoop.log.dir/,AVAILABLE}
2024-11-17T12:49:53,041 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@14628127{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-17T12:49:53,133 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@76be211{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/java.io.tmpdir/jetty-localhost-46733-hadoop-hdfs-3_4_1-tests_jar-_-any-8127384295548954084/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:49:53,134 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c2fc043{HTTP/1.1, (http/1.1)}{localhost:46733}
2024-11-17T12:49:53,134 INFO  [Time-limited test {}] server.Server(415): Started @288375ms
2024-11-17T12:49:53,135 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-17T12:49:53,694 WARN  [Thread-2497 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/cluster_f4408350-8688-a6d1-a2d2-82f46e2d86c0/data/data1/current/BP-447412456-172.17.0.2-1731847792369/current, will proceed with Du for space computation calculation, 
2024-11-17T12:49:53,694 WARN  [Thread-2498 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/cluster_f4408350-8688-a6d1-a2d2-82f46e2d86c0/data/data2/current/BP-447412456-172.17.0.2-1731847792369/current, will proceed with Du for space computation calculation, 
2024-11-17T12:49:53,715 WARN  [Thread-2461 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T12:49:53,717 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x46f013a6a061a3ff with lease ID 0xb3ea05723dba2475: Processing first storage report for DS-30c79915-be45-4acf-b864-5909879189f3 from datanode DatanodeRegistration(127.0.0.1:45341, datanodeUuid=32ba399b-cbb6-4348-8c0b-8d94cf1df6a1, infoPort=36081, infoSecurePort=0, ipcPort=38287, storageInfo=lv=-57;cid=testClusterID;nsid=1584183798;c=1731847792369)
2024-11-17T12:49:53,717 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x46f013a6a061a3ff with lease ID 0xb3ea05723dba2475: from storage DS-30c79915-be45-4acf-b864-5909879189f3 node DatanodeRegistration(127.0.0.1:45341, datanodeUuid=32ba399b-cbb6-4348-8c0b-8d94cf1df6a1, infoPort=36081, infoSecurePort=0, ipcPort=38287, storageInfo=lv=-57;cid=testClusterID;nsid=1584183798;c=1731847792369), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:49:53,717 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x46f013a6a061a3ff with lease ID 0xb3ea05723dba2475: Processing first storage report for DS-87a74a91-bb8d-4ee0-9ee7-6a36e4c37a12 from datanode DatanodeRegistration(127.0.0.1:45341, datanodeUuid=32ba399b-cbb6-4348-8c0b-8d94cf1df6a1, infoPort=36081, infoSecurePort=0, ipcPort=38287, storageInfo=lv=-57;cid=testClusterID;nsid=1584183798;c=1731847792369)
2024-11-17T12:49:53,717 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x46f013a6a061a3ff with lease ID 0xb3ea05723dba2475: from storage DS-87a74a91-bb8d-4ee0-9ee7-6a36e4c37a12 node DatanodeRegistration(127.0.0.1:45341, datanodeUuid=32ba399b-cbb6-4348-8c0b-8d94cf1df6a1, infoPort=36081, infoSecurePort=0, ipcPort=38287, storageInfo=lv=-57;cid=testClusterID;nsid=1584183798;c=1731847792369), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:49:53,832 WARN  [Thread-2508 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/cluster_f4408350-8688-a6d1-a2d2-82f46e2d86c0/data/data3/current/BP-447412456-172.17.0.2-1731847792369/current, will proceed with Du for space computation calculation, 
2024-11-17T12:49:53,832 WARN  [Thread-2509 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/cluster_f4408350-8688-a6d1-a2d2-82f46e2d86c0/data/data4/current/BP-447412456-172.17.0.2-1731847792369/current, will proceed with Du for space computation calculation, 
2024-11-17T12:49:53,854 WARN  [Thread-2484 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-17T12:49:53,856 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1c874d4918562a73 with lease ID 0xb3ea05723dba2476: Processing first storage report for DS-ad4ceba7-e2d7-4843-9bbf-54faa8f5856f from datanode DatanodeRegistration(127.0.0.1:35225, datanodeUuid=115ad31e-6643-4e06-b334-37e5800d61bf, infoPort=34867, infoSecurePort=0, ipcPort=38115, storageInfo=lv=-57;cid=testClusterID;nsid=1584183798;c=1731847792369)
2024-11-17T12:49:53,856 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1c874d4918562a73 with lease ID 0xb3ea05723dba2476: from storage DS-ad4ceba7-e2d7-4843-9bbf-54faa8f5856f node DatanodeRegistration(127.0.0.1:35225, datanodeUuid=115ad31e-6643-4e06-b334-37e5800d61bf, infoPort=34867, infoSecurePort=0, ipcPort=38115, storageInfo=lv=-57;cid=testClusterID;nsid=1584183798;c=1731847792369), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:49:53,856 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1c874d4918562a73 with lease ID 0xb3ea05723dba2476: Processing first storage report for DS-0c7dae01-6f18-497a-93cb-8ef02e40cfcd from datanode DatanodeRegistration(127.0.0.1:35225, datanodeUuid=115ad31e-6643-4e06-b334-37e5800d61bf, infoPort=34867, infoSecurePort=0, ipcPort=38115, storageInfo=lv=-57;cid=testClusterID;nsid=1584183798;c=1731847792369)
2024-11-17T12:49:53,856 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1c874d4918562a73 with lease ID 0xb3ea05723dba2476: from storage DS-0c7dae01-6f18-497a-93cb-8ef02e40cfcd node DatanodeRegistration(127.0.0.1:35225, datanodeUuid=115ad31e-6643-4e06-b334-37e5800d61bf, infoPort=34867, infoSecurePort=0, ipcPort=38115, storageInfo=lv=-57;cid=testClusterID;nsid=1584183798;c=1731847792369), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-17T12:49:53,859 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c
2024-11-17T12:49:53,869 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:53,869 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:53,877 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/cluster_f4408350-8688-a6d1-a2d2-82f46e2d86c0/zookeeper_0, clientPort=65257, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/cluster_f4408350-8688-a6d1-a2d2-82f46e2d86c0/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/cluster_f4408350-8688-a6d1-a2d2-82f46e2d86c0/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-17T12:49:53,878 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=65257
2024-11-17T12:49:53,878 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:49:53,880 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:49:53,890 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35225 is added to blk_1073741825_1001 (size=7)
2024-11-17T12:49:53,891 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45341 is added to blk_1073741825_1001 (size=7)
2024-11-17T12:49:53,892 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653 with version=8
2024-11-17T12:49:53,892 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:38459/user/jenkins/test-data/2e55bbe5-7623-029a-e8ad-5288eb6de88f/hbase-staging
2024-11-17T12:49:53,894 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/0d751a622496:0 server-side Connection retries=45
2024-11-17T12:49:53,894 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:49:53,894 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T12:49:53,894 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T12:49:53,894 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:49:53,894 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T12:49:53,894 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-17T12:49:53,894 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T12:49:53,895 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39839
2024-11-17T12:49:53,896 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39839 connecting to ZooKeeper ensemble=127.0.0.1:65257
2024-11-17T12:49:53,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:398390x0, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T12:49:53,945 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39839-0x1014a21bf980000 connected
2024-11-17T12:49:54,021 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:49:54,023 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:49:54,025 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:49:54,025 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653, hbase.cluster.distributed=false
2024-11-17T12:49:54,027 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T12:49:54,028 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39839
2024-11-17T12:49:54,028 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39839
2024-11-17T12:49:54,028 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39839
2024-11-17T12:49:54,029 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39839
2024-11-17T12:49:54,029 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39839
2024-11-17T12:49:54,046 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/0d751a622496:0 server-side Connection retries=45
2024-11-17T12:49:54,046 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:49:54,046 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-17T12:49:54,046 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-17T12:49:54,046 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-17T12:49:54,046 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-17T12:49:54,046 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-17T12:49:54,046 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-17T12:49:54,047 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33699
2024-11-17T12:49:54,047 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33699 connecting to ZooKeeper ensemble=127.0.0.1:65257
2024-11-17T12:49:54,048 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:49:54,049 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:49:54,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:336990x0, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-17T12:49:54,063 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33699-0x1014a21bf980001 connected
2024-11-17T12:49:54,063 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33699-0x1014a21bf980001, quorum=127.0.0.1:65257, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:49:54,063 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-17T12:49:54,064 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-17T12:49:54,065 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33699-0x1014a21bf980001, quorum=127.0.0.1:65257, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-17T12:49:54,066 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33699-0x1014a21bf980001, quorum=127.0.0.1:65257, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-17T12:49:54,066 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33699
2024-11-17T12:49:54,066 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33699
2024-11-17T12:49:54,067 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33699
2024-11-17T12:49:54,067 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33699
2024-11-17T12:49:54,067 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33699
2024-11-17T12:49:54,081 DEBUG [M:0;0d751a622496:39839 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0d751a622496:39839
2024-11-17T12:49:54,082 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0d751a622496,39839,1731847793894
2024-11-17T12:49:54,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:49:54,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33699-0x1014a21bf980001, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:49:54,093 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0d751a622496,39839,1731847793894
2024-11-17T12:49:54,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:54,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33699-0x1014a21bf980001, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-17T12:49:54,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33699-0x1014a21bf980001, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:54,105 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-17T12:49:54,105 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0d751a622496,39839,1731847793894 from backup master directory
2024-11-17T12:49:54,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0d751a622496,39839,1731847793894
2024-11-17T12:49:54,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33699-0x1014a21bf980001, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:49:54,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-17T12:49:54,112 WARN  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T12:49:54,112 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0d751a622496,39839,1731847793894
2024-11-17T12:49:54,116 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/hbase.id] with ID: e2ac0c10-001f-4770-bd10-72cd8b45216b
2024-11-17T12:49:54,116 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/.tmp/hbase.id
2024-11-17T12:49:54,120 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45341 is added to blk_1073741826_1002 (size=42)
2024-11-17T12:49:54,121 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35225 is added to blk_1073741826_1002 (size=42)
2024-11-17T12:49:54,121 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/.tmp/hbase.id]:[hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/hbase.id]
2024-11-17T12:49:54,131 INFO  [master/0d751a622496:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:49:54,131 INFO  [master/0d751a622496:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-17T12:49:54,132 INFO  [master/0d751a622496:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-17T12:49:54,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33699-0x1014a21bf980001, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:54,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:54,149 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35225 is added to blk_1073741827_1003 (size=196)
2024-11-17T12:49:54,149 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45341 is added to blk_1073741827_1003 (size=196)
2024-11-17T12:49:54,149 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-17T12:49:54,150 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-17T12:49:54,150 INFO  [master/0d751a622496:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T12:49:54,157 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45341 is added to blk_1073741828_1004 (size=1189)
2024-11-17T12:49:54,158 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35225 is added to blk_1073741828_1004 (size=1189)
2024-11-17T12:49:54,159 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/data/master/store
2024-11-17T12:49:54,168 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35225 is added to blk_1073741829_1005 (size=34)
2024-11-17T12:49:54,168 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45341 is added to blk_1073741829_1005 (size=34)
2024-11-17T12:49:54,169 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:49:54,169 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T12:49:54,169 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:49:54,169 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:49:54,169 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T12:49:54,169 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:49:54,169 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:49:54,169 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731847794169Disabling compacts and flushes for region at 1731847794169Disabling writes for close at 1731847794169Writing region close event to WAL at 1731847794169Closed at 1731847794169
2024-11-17T12:49:54,170 WARN  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/data/master/store/.initializing
2024-11-17T12:49:54,170 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/WALs/0d751a622496,39839,1731847793894
2024-11-17T12:49:54,172 INFO  [master/0d751a622496:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C39839%2C1731847793894, suffix=, logDir=hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/WALs/0d751a622496,39839,1731847793894, archiveDir=hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/oldWALs, maxLogs=10
2024-11-17T12:49:54,172 INFO  [master/0d751a622496:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C39839%2C1731847793894.1731847794172
2024-11-17T12:49:54,180 INFO  [master/0d751a622496:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/WALs/0d751a622496,39839,1731847793894/0d751a622496%2C39839%2C1731847793894.1731847794172
2024-11-17T12:49:54,187 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36081:36081),(127.0.0.1/127.0.0.1:34867:34867)]
2024-11-17T12:49:54,189 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-17T12:49:54,190 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:49:54,190 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:49:54,190 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:49:54,191 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:49:54,192 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-17T12:49:54,192 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:54,192 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:49:54,193 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:49:54,194 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-17T12:49:54,194 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:54,194 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:49:54,194 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:49:54,195 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-17T12:49:54,195 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:54,196 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:49:54,196 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-17T12:49:54,197 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-17T12:49:54,197 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:54,197 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-17T12:49:54,197 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:49:54,198 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:49:54,198 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:49:54,199 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:49:54,199 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:49:54,200 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-17T12:49:54,200 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-17T12:49:54,202 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T12:49:54,202 INFO  [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=837854, jitterRate=0.06538672745227814}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-17T12:49:54,202 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731847794190Initializing all the Stores at 1731847794191 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847794191Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847794191Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847794191Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847794191Cleaning up temporary data from old regions at 1731847794199 (+8 ms)Region opened successfully at 1731847794202 (+3 ms)
2024-11-17T12:49:54,203 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-17T12:49:54,205 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@370bcfd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0d751a622496/172.17.0.2:0
2024-11-17T12:49:54,205 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-17T12:49:54,206 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-17T12:49:54,206 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-17T12:49:54,206 INFO  [master/0d751a622496:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-17T12:49:54,206 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-17T12:49:54,206 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-17T12:49:54,206 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-17T12:49:54,208 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-17T12:49:54,208 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-17T12:49:54,242 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-17T12:49:54,243 INFO  [master/0d751a622496:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-17T12:49:54,243 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-17T12:49:54,254 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-17T12:49:54,254 INFO  [master/0d751a622496:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-17T12:49:54,255 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-17T12:49:54,262 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-17T12:49:54,263 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-17T12:49:54,271 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-17T12:49:54,273 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-17T12:49:54,279 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-17T12:49:54,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33699-0x1014a21bf980001, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T12:49:54,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-17T12:49:54,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:54,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33699-0x1014a21bf980001, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:54,288 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0d751a622496,39839,1731847793894, sessionid=0x1014a21bf980000, setting cluster-up flag (Was=false)
2024-11-17T12:49:54,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:54,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33699-0x1014a21bf980001, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:54,329 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-17T12:49:54,330 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0d751a622496,39839,1731847793894
2024-11-17T12:49:54,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:54,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33699-0x1014a21bf980001, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:54,371 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-17T12:49:54,372 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0d751a622496,39839,1731847793894
2024-11-17T12:49:54,373 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-17T12:49:54,375 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-17T12:49:54,375 INFO  [master/0d751a622496:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-17T12:49:54,375 INFO  [master/0d751a622496:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-17T12:49:54,375 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0d751a622496,39839,1731847793894
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-17T12:49:54,377 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:49:54,377 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:49:54,377 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:49:54,377 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5
2024-11-17T12:49:54,377 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0d751a622496:0, corePoolSize=10, maxPoolSize=10
2024-11-17T12:49:54,377 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:49:54,377 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0d751a622496:0, corePoolSize=2, maxPoolSize=2
2024-11-17T12:49:54,377 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:49:54,378 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731847824378
2024-11-17T12:49:54,378 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-17T12:49:54,378 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-17T12:49:54,378 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-17T12:49:54,378 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-17T12:49:54,378 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-17T12:49:54,378 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-17T12:49:54,378 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:54,378 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T12:49:54,379 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-17T12:49:54,379 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-17T12:49:54,379 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-17T12:49:54,379 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-17T12:49:54,379 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-17T12:49:54,379 INFO  [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-17T12:49:54,379 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.large.0-1731847794379,5,FailOnTimeoutGroup]
2024-11-17T12:49:54,379 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.small.0-1731847794379,5,FailOnTimeoutGroup]
2024-11-17T12:49:54,379 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:54,379 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-17T12:49:54,379 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:54,379 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:54,379 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:54,380 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-17T12:49:54,385 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45341 is added to blk_1073741831_1007 (size=1321)
2024-11-17T12:49:54,385 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35225 is added to blk_1073741831_1007 (size=1321)
2024-11-17T12:49:54,386 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-17T12:49:54,386 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653
2024-11-17T12:49:54,392 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45341 is added to blk_1073741832_1008 (size=32)
2024-11-17T12:49:54,392 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35225 is added to blk_1073741832_1008 (size=32)
2024-11-17T12:49:54,393 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:49:54,394 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T12:49:54,395 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T12:49:54,395 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:54,395 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:49:54,396 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T12:49:54,397 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T12:49:54,397 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:54,397 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:49:54,397 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T12:49:54,398 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T12:49:54,398 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:54,399 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:49:54,399 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T12:49:54,400 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T12:49:54,400 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:54,400 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:49:54,400 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T12:49:54,401 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/data/hbase/meta/1588230740
2024-11-17T12:49:54,401 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/data/hbase/meta/1588230740
2024-11-17T12:49:54,402 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T12:49:54,402 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T12:49:54,402 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T12:49:54,403 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T12:49:54,405 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-17T12:49:54,405 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=767157, jitterRate=-0.024509727954864502}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T12:49:54,406 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731847794393Initializing all the Stores at 1731847794393Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847794393Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847794393Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847794393Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847794393Cleaning up temporary data from old regions at 1731847794402 (+9 ms)Region opened successfully at 1731847794406 (+4 ms)
2024-11-17T12:49:54,406 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T12:49:54,406 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T12:49:54,406 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T12:49:54,406 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T12:49:54,406 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T12:49:54,406 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T12:49:54,406 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731847794406Disabling compacts and flushes for region at 1731847794406Disabling writes for close at 1731847794406Writing region close event to WAL at 1731847794406Closed at 1731847794406
2024-11-17T12:49:54,407 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T12:49:54,407 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-17T12:49:54,407 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-17T12:49:54,408 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T12:49:54,409 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-17T12:49:54,470 INFO  [RS:0;0d751a622496:33699 {}] regionserver.HRegionServer(746): ClusterId : e2ac0c10-001f-4770-bd10-72cd8b45216b
2024-11-17T12:49:54,470 DEBUG [RS:0;0d751a622496:33699 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-17T12:49:54,498 DEBUG [RS:0;0d751a622496:33699 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-17T12:49:54,498 DEBUG [RS:0;0d751a622496:33699 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-17T12:49:54,510 DEBUG [RS:0;0d751a622496:33699 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-17T12:49:54,511 DEBUG [RS:0;0d751a622496:33699 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29b6d1c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0d751a622496/172.17.0.2:0
2024-11-17T12:49:54,525 DEBUG [RS:0;0d751a622496:33699 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0d751a622496:33699
2024-11-17T12:49:54,526 INFO  [RS:0;0d751a622496:33699 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-17T12:49:54,526 INFO  [RS:0;0d751a622496:33699 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-17T12:49:54,526 DEBUG [RS:0;0d751a622496:33699 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-17T12:49:54,526 INFO  [RS:0;0d751a622496:33699 {}] regionserver.HRegionServer(2659): reportForDuty to master=0d751a622496,39839,1731847793894 with port=33699, startcode=1731847794046
2024-11-17T12:49:54,527 DEBUG [RS:0;0d751a622496:33699 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-17T12:49:54,528 INFO  [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57661, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService
2024-11-17T12:49:54,529 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39839 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0d751a622496,33699,1731847794046
2024-11-17T12:49:54,529 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39839 {}] master.ServerManager(517): Registering regionserver=0d751a622496,33699,1731847794046
2024-11-17T12:49:54,530 DEBUG [RS:0;0d751a622496:33699 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653
2024-11-17T12:49:54,530 DEBUG [RS:0;0d751a622496:33699 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40567
2024-11-17T12:49:54,530 DEBUG [RS:0;0d751a622496:33699 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-17T12:49:54,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T12:49:54,538 DEBUG [RS:0;0d751a622496:33699 {}] zookeeper.ZKUtil(111): regionserver:33699-0x1014a21bf980001, quorum=127.0.0.1:65257, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0d751a622496,33699,1731847794046
2024-11-17T12:49:54,538 WARN  [RS:0;0d751a622496:33699 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-17T12:49:54,538 INFO  [RS:0;0d751a622496:33699 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T12:49:54,538 DEBUG [RS:0;0d751a622496:33699 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/WALs/0d751a622496,33699,1731847794046
2024-11-17T12:49:54,538 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0d751a622496,33699,1731847794046]
2024-11-17T12:49:54,541 INFO  [RS:0;0d751a622496:33699 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-17T12:49:54,542 INFO  [RS:0;0d751a622496:33699 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-17T12:49:54,543 INFO  [RS:0;0d751a622496:33699 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-17T12:49:54,543 INFO  [RS:0;0d751a622496:33699 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:54,543 INFO  [RS:0;0d751a622496:33699 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-17T12:49:54,544 INFO  [RS:0;0d751a622496:33699 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-17T12:49:54,544 INFO  [RS:0;0d751a622496:33699 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:54,544 DEBUG [RS:0;0d751a622496:33699 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:49:54,544 DEBUG [RS:0;0d751a622496:33699 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:49:54,544 DEBUG [RS:0;0d751a622496:33699 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:49:54,544 DEBUG [RS:0;0d751a622496:33699 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:49:54,544 DEBUG [RS:0;0d751a622496:33699 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:49:54,544 DEBUG [RS:0;0d751a622496:33699 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0, corePoolSize=2, maxPoolSize=2
2024-11-17T12:49:54,544 DEBUG [RS:0;0d751a622496:33699 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:49:54,544 DEBUG [RS:0;0d751a622496:33699 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:49:54,544 DEBUG [RS:0;0d751a622496:33699 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:49:54,544 DEBUG [RS:0;0d751a622496:33699 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:49:54,544 DEBUG [RS:0;0d751a622496:33699 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:49:54,544 DEBUG [RS:0;0d751a622496:33699 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1
2024-11-17T12:49:54,544 DEBUG [RS:0;0d751a622496:33699 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0d751a622496:0, corePoolSize=3, maxPoolSize=3
2024-11-17T12:49:54,544 DEBUG [RS:0;0d751a622496:33699 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0, corePoolSize=3, maxPoolSize=3
2024-11-17T12:49:54,545 INFO  [RS:0;0d751a622496:33699 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:54,545 INFO  [RS:0;0d751a622496:33699 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:54,545 INFO  [RS:0;0d751a622496:33699 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:54,545 INFO  [RS:0;0d751a622496:33699 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:54,545 INFO  [RS:0;0d751a622496:33699 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:54,545 INFO  [RS:0;0d751a622496:33699 {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,33699,1731847794046-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T12:49:54,559 WARN  [0d751a622496:39839 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-11-17T12:49:54,561 INFO  [RS:0;0d751a622496:33699 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-17T12:49:54,561 INFO  [RS:0;0d751a622496:33699 {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,33699,1731847794046-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:54,561 INFO  [RS:0;0d751a622496:33699 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:54,561 INFO  [RS:0;0d751a622496:33699 {}] regionserver.Replication(171): 0d751a622496,33699,1731847794046 started
2024-11-17T12:49:54,574 INFO  [RS:0;0d751a622496:33699 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:54,574 INFO  [RS:0;0d751a622496:33699 {}] regionserver.HRegionServer(1482): Serving as 0d751a622496,33699,1731847794046, RpcServer on 0d751a622496/172.17.0.2:33699, sessionid=0x1014a21bf980001
2024-11-17T12:49:54,574 DEBUG [RS:0;0d751a622496:33699 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-17T12:49:54,574 DEBUG [RS:0;0d751a622496:33699 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0d751a622496,33699,1731847794046
2024-11-17T12:49:54,574 DEBUG [RS:0;0d751a622496:33699 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0d751a622496,33699,1731847794046'
2024-11-17T12:49:54,574 DEBUG [RS:0;0d751a622496:33699 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-17T12:49:54,574 DEBUG [RS:0;0d751a622496:33699 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-17T12:49:54,575 DEBUG [RS:0;0d751a622496:33699 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-17T12:49:54,575 DEBUG [RS:0;0d751a622496:33699 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-17T12:49:54,575 DEBUG [RS:0;0d751a622496:33699 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0d751a622496,33699,1731847794046
2024-11-17T12:49:54,575 DEBUG [RS:0;0d751a622496:33699 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0d751a622496,33699,1731847794046'
2024-11-17T12:49:54,575 DEBUG [RS:0;0d751a622496:33699 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-17T12:49:54,575 DEBUG [RS:0;0d751a622496:33699 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-17T12:49:54,576 DEBUG [RS:0;0d751a622496:33699 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-17T12:49:54,576 INFO  [RS:0;0d751a622496:33699 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-17T12:49:54,576 INFO  [RS:0;0d751a622496:33699 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-17T12:49:54,679 INFO  [RS:0;0d751a622496:33699 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C33699%2C1731847794046, suffix=, logDir=hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/WALs/0d751a622496,33699,1731847794046, archiveDir=hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/oldWALs, maxLogs=32
2024-11-17T12:49:54,680 INFO  [RS:0;0d751a622496:33699 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C33699%2C1731847794046.1731847794679
2024-11-17T12:49:54,688 INFO  [RS:0;0d751a622496:33699 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/WALs/0d751a622496,33699,1731847794046/0d751a622496%2C33699%2C1731847794046.1731847794679
2024-11-17T12:49:54,689 DEBUG [RS:0;0d751a622496:33699 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34867:34867),(127.0.0.1/127.0.0.1:36081:36081)]
2024-11-17T12:49:54,810 DEBUG [0d751a622496:39839 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-17T12:49:54,810 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0d751a622496,33699,1731847794046
2024-11-17T12:49:54,812 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0d751a622496,33699,1731847794046, state=OPENING
2024-11-17T12:49:54,841 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-17T12:49:54,869 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,43029,1731847612705/0d751a622496%2C43029%2C1731847612705.meta.1731847613728.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:54,869 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42079/user/jenkins/test-data/2611c35b-26d1-cbdd-feb2-e45d60b8fdc8/WALs/0d751a622496,37689,1731847613976/0d751a622496%2C37689%2C1731847613976.1731847614224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-17T12:49:54,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:54,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33699-0x1014a21bf980001, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:54,888 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-17T12:49:54,888 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:49:54,888 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:49:54,888 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0d751a622496,33699,1731847794046}]
2024-11-17T12:49:55,043 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-17T12:49:55,046 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47027, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-17T12:49:55,051 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-17T12:49:55,051 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T12:49:55,053 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C33699%2C1731847794046.meta, suffix=.meta, logDir=hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/WALs/0d751a622496,33699,1731847794046, archiveDir=hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/oldWALs, maxLogs=32
2024-11-17T12:49:55,053 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 0d751a622496%2C33699%2C1731847794046.meta.1731847795053.meta
2024-11-17T12:49:55,058 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/WALs/0d751a622496,33699,1731847794046/0d751a622496%2C33699%2C1731847794046.meta.1731847795053.meta
2024-11-17T12:49:55,059 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36081:36081),(127.0.0.1/127.0.0.1:34867:34867)]
2024-11-17T12:49:55,060 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-17T12:49:55,060 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-17T12:49:55,061 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-17T12:49:55,061 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-17T12:49:55,061 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-17T12:49:55,061 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-17T12:49:55,061 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-17T12:49:55,061 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-17T12:49:55,063 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-17T12:49:55,064 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-17T12:49:55,064 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:55,064 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:49:55,064 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-17T12:49:55,065 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-17T12:49:55,065 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:55,066 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:49:55,066 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-17T12:49:55,066 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-17T12:49:55,066 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:55,067 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:49:55,067 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-17T12:49:55,068 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-17T12:49:55,068 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-17T12:49:55,068 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-17T12:49:55,068 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-17T12:49:55,069 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/data/hbase/meta/1588230740
2024-11-17T12:49:55,070 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/data/hbase/meta/1588230740
2024-11-17T12:49:55,071 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-17T12:49:55,071 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-17T12:49:55,071 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-17T12:49:55,072 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-17T12:49:55,073 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=727170, jitterRate=-0.0753556489944458}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-17T12:49:55,073 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-17T12:49:55,073 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731847795061Writing region info on filesystem at 1731847795061Initializing all the Stores at 1731847795062 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847795062Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847795063 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847795063Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847795063Cleaning up temporary data from old regions at 1731847795071 (+8 ms)Running coprocessor post-open hooks at 1731847795073 (+2 ms)Region opened successfully at 1731847795073
2024-11-17T12:49:55,074 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731847795042
2024-11-17T12:49:55,076 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-17T12:49:55,076 INFO  [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-17T12:49:55,076 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0d751a622496,33699,1731847794046
2024-11-17T12:49:55,077 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0d751a622496,33699,1731847794046, state=OPEN
2024-11-17T12:49:55,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T12:49:55,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33699-0x1014a21bf980001, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-17T12:49:55,120 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0d751a622496,33699,1731847794046
2024-11-17T12:49:55,120 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:49:55,120 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-17T12:49:55,122 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-17T12:49:55,122 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0d751a622496,33699,1731847794046 in 232 msec
2024-11-17T12:49:55,124 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-17T12:49:55,125 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 715 msec
2024-11-17T12:49:55,125 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-17T12:49:55,125 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-17T12:49:55,127 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T12:49:55,127 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0d751a622496,33699,1731847794046, seqNum=-1]
2024-11-17T12:49:55,127 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T12:49:55,128 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44177, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T12:49:55,132 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 757 msec
2024-11-17T12:49:55,132 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731847795132, completionTime=-1
2024-11-17T12:49:55,133 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-17T12:49:55,133 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-11-17T12:49:55,134 INFO  [master/0d751a622496:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-11-17T12:49:55,134 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731847855134
2024-11-17T12:49:55,134 INFO  [master/0d751a622496:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731847915134
2024-11-17T12:49:55,134 INFO  [master/0d751a622496:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-11-17T12:49:55,135 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,39839,1731847793894-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:55,135 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,39839,1731847793894-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:55,135 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,39839,1731847793894-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:55,135 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0d751a622496:39839, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:55,135 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:55,135 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:55,137 DEBUG [master/0d751a622496:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-17T12:49:55,139 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.026sec
2024-11-17T12:49:55,139 INFO  [master/0d751a622496:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-17T12:49:55,139 INFO  [master/0d751a622496:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-17T12:49:55,139 INFO  [master/0d751a622496:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-17T12:49:55,139 INFO  [master/0d751a622496:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-17T12:49:55,139 INFO  [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-17T12:49:55,139 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,39839,1731847793894-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-17T12:49:55,139 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,39839,1731847793894-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-17T12:49:55,141 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-17T12:49:55,141 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-17T12:49:55,141 INFO  [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,39839,1731847793894-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-17T12:49:55,170 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e3053e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T12:49:55,170 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0d751a622496,39839,-1 for getting cluster id
2024-11-17T12:49:55,170 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-17T12:49:55,172 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e2ac0c10-001f-4770-bd10-72cd8b45216b'
2024-11-17T12:49:55,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-17T12:49:55,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e2ac0c10-001f-4770-bd10-72cd8b45216b"

2024-11-17T12:49:55,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@162e873b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T12:49:55,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0d751a622496,39839,-1]
2024-11-17T12:49:55,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-17T12:49:55,173 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:49:55,174 INFO  [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35418, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-17T12:49:55,175 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f9361ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-17T12:49:55,175 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-17T12:49:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0d751a622496,33699,1731847794046, seqNum=-1]
2024-11-17T12:49:55,177 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-17T12:49:55,178 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59076, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-17T12:49:55,180 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0d751a622496,39839,1731847793894
2024-11-17T12:49:55,181 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-17T12:49:55,184 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-17T12:49:55,185 INFO  [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-17T12:49:55,187 INFO  [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/WALs/test.com,8080,1, archiveDir=hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/oldWALs, maxLogs=32
2024-11-17T12:49:55,188 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1731847795188
2024-11-17T12:49:55,195 INFO  [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/WALs/test.com,8080,1/test.com%2C8080%2C1.1731847795188
2024-11-17T12:49:55,196 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34867:34867),(127.0.0.1/127.0.0.1:36081:36081)]
2024-11-17T12:49:55,198 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1731847795197
2024-11-17T12:49:55,203 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:55,203 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:55,203 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:55,203 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:55,203 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:55,203 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/WALs/test.com,8080,1/test.com%2C8080%2C1.1731847795188 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/WALs/test.com,8080,1/test.com%2C8080%2C1.1731847795197
2024-11-17T12:49:55,204 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34867:34867),(127.0.0.1/127.0.0.1:36081:36081)]
2024-11-17T12:49:55,204 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/WALs/test.com,8080,1/test.com%2C8080%2C1.1731847795188 is not closed yet, will try archiving it next time
2024-11-17T12:49:55,205 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45341 is added to blk_1073741835_1011 (size=93)
2024-11-17T12:49:55,205 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:55,205 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:55,205 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35225 is added to blk_1073741835_1011 (size=93)
2024-11-17T12:49:55,205 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:55,205 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:55,206 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/WALs/test.com,8080,1/test.com%2C8080%2C1.1731847795188 to hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/oldWALs/test.com%2C8080%2C1.1731847795188
2024-11-17T12:49:55,206 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:55,208 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45341 is added to blk_1073741836_1012 (size=93)
2024-11-17T12:49:55,208 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35225 is added to blk_1073741836_1012 (size=93)
2024-11-17T12:49:55,212 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/oldWALs
2024-11-17T12:49:55,212 INFO  [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1731847795197)
2024-11-17T12:49:55,212 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-17T12:49:55,212 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T12:49:55,212 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T12:49:55,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:49:55,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:49:55,213 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-17T12:49:55,213 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-17T12:49:55,213 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1092902566, stopped=false
2024-11-17T12:49:55,213 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0d751a622496,39839,1731847793894
2024-11-17T12:49:55,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T12:49:55,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33699-0x1014a21bf980001, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-17T12:49:55,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33699-0x1014a21bf980001, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:55,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:55,229 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T12:49:55,229 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-17T12:49:55,229 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T12:49:55,229 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:49:55,229 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33699-0x1014a21bf980001, quorum=127.0.0.1:65257, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:49:55,229 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0d751a622496,33699,1731847794046' *****
2024-11-17T12:49:55,230 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-17T12:49:55,230 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-17T12:49:55,230 INFO  [RS:0;0d751a622496:33699 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-17T12:49:55,230 INFO  [RS:0;0d751a622496:33699 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-17T12:49:55,230 INFO  [RS:0;0d751a622496:33699 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-17T12:49:55,230 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-17T12:49:55,230 INFO  [RS:0;0d751a622496:33699 {}] regionserver.HRegionServer(959): stopping server 0d751a622496,33699,1731847794046
2024-11-17T12:49:55,230 INFO  [RS:0;0d751a622496:33699 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T12:49:55,230 INFO  [RS:0;0d751a622496:33699 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0d751a622496:33699.
2024-11-17T12:49:55,230 DEBUG [RS:0;0d751a622496:33699 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-17T12:49:55,230 DEBUG [RS:0;0d751a622496:33699 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:49:55,230 INFO  [RS:0;0d751a622496:33699 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-17T12:49:55,230 INFO  [RS:0;0d751a622496:33699 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-17T12:49:55,230 INFO  [RS:0;0d751a622496:33699 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-17T12:49:55,230 INFO  [RS:0;0d751a622496:33699 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-17T12:49:55,230 INFO  [RS:0;0d751a622496:33699 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-17T12:49:55,230 DEBUG [RS:0;0d751a622496:33699 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-17T12:49:55,230 DEBUG [RS:0;0d751a622496:33699 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-17T12:49:55,230 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-17T12:49:55,231 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-17T12:49:55,231 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-17T12:49:55,231 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-17T12:49:55,231 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-17T12:49:55,231 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-17T12:49:55,246 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/data/hbase/meta/1588230740/.tmp/ns/6f0ddad566d44450bca587dbc91ac01e is 43, key is default/ns:d/1731847795128/Put/seqid=0
2024-11-17T12:49:55,250 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45341 is added to blk_1073741837_1013 (size=5153)
2024-11-17T12:49:55,250 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35225 is added to blk_1073741837_1013 (size=5153)
2024-11-17T12:49:55,250 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/data/hbase/meta/1588230740/.tmp/ns/6f0ddad566d44450bca587dbc91ac01e
2024-11-17T12:49:55,255 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/data/hbase/meta/1588230740/.tmp/ns/6f0ddad566d44450bca587dbc91ac01e as hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/data/hbase/meta/1588230740/ns/6f0ddad566d44450bca587dbc91ac01e
2024-11-17T12:49:55,260 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/data/hbase/meta/1588230740/ns/6f0ddad566d44450bca587dbc91ac01e, entries=2, sequenceid=6, filesize=5.0 K
2024-11-17T12:49:55,261 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 30ms, sequenceid=6, compaction requested=false
2024-11-17T12:49:55,265 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-17T12:49:55,265 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-17T12:49:55,265 INFO  [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-17T12:49:55,265 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731847795230Running coprocessor pre-close hooks at 1731847795230Disabling compacts and flushes for region at 1731847795230Disabling writes for close at 1731847795231 (+1 ms)Obtaining lock to block concurrent updates at 1731847795231Preparing flush snapshotting stores in 1588230740 at 1731847795231Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1731847795231Flushing stores of hbase:meta,,1.1588230740 at 1731847795232 (+1 ms)Flushing 1588230740/ns: creating writer at 1731847795232Flushing 1588230740/ns: appending metadata at 1731847795246 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731847795246Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2794f123: reopening flushed file at 1731847795255 (+9 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 30ms, sequenceid=6, compaction requested=false at 1731847795261 (+6 ms)Writing region close event to WAL at 1731847795262 (+1 ms)Running coprocessor post-close hooks at 1731847795265 (+3 ms)Closed at 1731847795265
2024-11-17T12:49:55,266 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-17T12:49:55,431 INFO  [RS:0;0d751a622496:33699 {}] regionserver.HRegionServer(976): stopping server 0d751a622496,33699,1731847794046; all regions closed.
2024-11-17T12:49:55,432 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:55,432 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:55,433 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:55,433 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:55,433 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:55,436 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35225 is added to blk_1073741834_1010 (size=1152)
2024-11-17T12:49:55,436 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45341 is added to blk_1073741834_1010 (size=1152)
2024-11-17T12:49:55,440 DEBUG [RS:0;0d751a622496:33699 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/oldWALs
2024-11-17T12:49:55,440 INFO  [RS:0;0d751a622496:33699 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0d751a622496%2C33699%2C1731847794046.meta:.meta(num 1731847795053)
2024-11-17T12:49:55,440 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:55,440 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:55,441 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:55,441 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:55,441 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:55,443 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45341 is added to blk_1073741833_1009 (size=93)
2024-11-17T12:49:55,444 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35225 is added to blk_1073741833_1009 (size=93)
2024-11-17T12:49:55,446 DEBUG [RS:0;0d751a622496:33699 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/oldWALs
2024-11-17T12:49:55,446 INFO  [RS:0;0d751a622496:33699 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 0d751a622496%2C33699%2C1731847794046:(num 1731847794679)
2024-11-17T12:49:55,446 DEBUG [RS:0;0d751a622496:33699 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-17T12:49:55,446 INFO  [RS:0;0d751a622496:33699 {}] regionserver.LeaseManager(133): Closed leases
2024-11-17T12:49:55,446 INFO  [RS:0;0d751a622496:33699 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T12:49:55,447 INFO  [RS:0;0d751a622496:33699 {}] hbase.ChoreService(370): Chore service for: regionserver/0d751a622496:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-17T12:49:55,447 INFO  [RS:0;0d751a622496:33699 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T12:49:55,447 INFO  [regionserver/0d751a622496:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T12:49:55,447 INFO  [RS:0;0d751a622496:33699 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33699
2024-11-17T12:49:55,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33699-0x1014a21bf980001, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0d751a622496,33699,1731847794046
2024-11-17T12:49:55,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-17T12:49:55,476 INFO  [RS:0;0d751a622496:33699 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T12:49:55,487 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0d751a622496,33699,1731847794046]
2024-11-17T12:49:55,495 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0d751a622496,33699,1731847794046 already deleted, retry=false
2024-11-17T12:49:55,495 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0d751a622496,33699,1731847794046 expired; onlineServers=0
2024-11-17T12:49:55,495 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0d751a622496,39839,1731847793894' *****
2024-11-17T12:49:55,495 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-17T12:49:55,496 INFO  [M:0;0d751a622496:39839 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-17T12:49:55,496 INFO  [M:0;0d751a622496:39839 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-17T12:49:55,496 DEBUG [M:0;0d751a622496:39839 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-17T12:49:55,496 DEBUG [M:0;0d751a622496:39839 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-17T12:49:55,496 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-17T12:49:55,496 DEBUG [master/0d751a622496:0:becomeActiveMaster-HFileCleaner.small.0-1731847794379 {}] cleaner.HFileCleaner(306): Exit Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.small.0-1731847794379,5,FailOnTimeoutGroup]
2024-11-17T12:49:55,496 DEBUG [master/0d751a622496:0:becomeActiveMaster-HFileCleaner.large.0-1731847794379 {}] cleaner.HFileCleaner(306): Exit Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.large.0-1731847794379,5,FailOnTimeoutGroup]
2024-11-17T12:49:55,496 INFO  [M:0;0d751a622496:39839 {}] hbase.ChoreService(370): Chore service for: master/0d751a622496:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-17T12:49:55,496 INFO  [M:0;0d751a622496:39839 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-17T12:49:55,496 DEBUG [M:0;0d751a622496:39839 {}] master.HMaster(1795): Stopping service threads
2024-11-17T12:49:55,496 INFO  [M:0;0d751a622496:39839 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-17T12:49:55,496 INFO  [M:0;0d751a622496:39839 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-17T12:49:55,496 INFO  [M:0;0d751a622496:39839 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-17T12:49:55,496 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-17T12:49:55,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-17T12:49:55,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-17T12:49:55,504 DEBUG [M:0;0d751a622496:39839 {}] zookeeper.ZKUtil(347): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-17T12:49:55,504 WARN  [M:0;0d751a622496:39839 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-17T12:49:55,505 INFO  [M:0;0d751a622496:39839 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/.lastflushedseqids
2024-11-17T12:49:55,510 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45341 is added to blk_1073741838_1014 (size=99)
2024-11-17T12:49:55,510 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35225 is added to blk_1073741838_1014 (size=99)
2024-11-17T12:49:55,510 INFO  [M:0;0d751a622496:39839 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-17T12:49:55,510 INFO  [M:0;0d751a622496:39839 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-17T12:49:55,510 DEBUG [M:0;0d751a622496:39839 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-17T12:49:55,510 INFO  [M:0;0d751a622496:39839 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:49:55,510 DEBUG [M:0;0d751a622496:39839 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:49:55,510 DEBUG [M:0;0d751a622496:39839 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-17T12:49:55,510 DEBUG [M:0;0d751a622496:39839 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:49:55,510 INFO  [M:0;0d751a622496:39839 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-17T12:49:55,527 DEBUG [M:0;0d751a622496:39839 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/14d000b270b74cd7b11f5bc7e210efc1 is 82, key is hbase:meta,,1/info:regioninfo/1731847795076/Put/seqid=0
2024-11-17T12:49:55,532 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45341 is added to blk_1073741839_1015 (size=5672)
2024-11-17T12:49:55,532 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35225 is added to blk_1073741839_1015 (size=5672)
2024-11-17T12:49:55,538 INFO  [M:0;0d751a622496:39839 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/14d000b270b74cd7b11f5bc7e210efc1
2024-11-17T12:49:55,565 DEBUG [M:0;0d751a622496:39839 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/98b181ca94414f10a0f9050422000992 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1731847795132/Put/seqid=0
2024-11-17T12:49:55,569 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35225 is added to blk_1073741840_1016 (size=5275)
2024-11-17T12:49:55,569 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45341 is added to blk_1073741840_1016 (size=5275)
2024-11-17T12:49:55,570 INFO  [M:0;0d751a622496:39839 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/98b181ca94414f10a0f9050422000992
2024-11-17T12:49:55,587 INFO  [RS:0;0d751a622496:33699 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T12:49:55,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33699-0x1014a21bf980001, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:49:55,588 INFO  [RS:0;0d751a622496:33699 {}] regionserver.HRegionServer(1031): Exiting; stopping=0d751a622496,33699,1731847794046; zookeeper connection closed.
2024-11-17T12:49:55,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33699-0x1014a21bf980001, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:49:55,588 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@38b26971 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@38b26971
2024-11-17T12:49:55,588 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-17T12:49:55,592 DEBUG [M:0;0d751a622496:39839 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ec5922d6f26b4b3ea02e3dcbe37ec3f9 is 69, key is 0d751a622496,33699,1731847794046/rs:state/1731847794529/Put/seqid=0
2024-11-17T12:49:55,596 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35225 is added to blk_1073741841_1017 (size=5156)
2024-11-17T12:49:55,596 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45341 is added to blk_1073741841_1017 (size=5156)
2024-11-17T12:49:55,596 INFO  [M:0;0d751a622496:39839 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ec5922d6f26b4b3ea02e3dcbe37ec3f9
2024-11-17T12:49:55,614 DEBUG [M:0;0d751a622496:39839 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/fba3bf304b4b4c49bc61cff34d640c29 is 52, key is load_balancer_on/state:d/1731847795183/Put/seqid=0
2024-11-17T12:49:55,618 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35225 is added to blk_1073741842_1018 (size=5056)
2024-11-17T12:49:55,618 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45341 is added to blk_1073741842_1018 (size=5056)
2024-11-17T12:49:55,619 INFO  [M:0;0d751a622496:39839 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/fba3bf304b4b4c49bc61cff34d640c29
2024-11-17T12:49:55,623 DEBUG [M:0;0d751a622496:39839 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/14d000b270b74cd7b11f5bc7e210efc1 as hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/14d000b270b74cd7b11f5bc7e210efc1
2024-11-17T12:49:55,627 INFO  [M:0;0d751a622496:39839 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/14d000b270b74cd7b11f5bc7e210efc1, entries=8, sequenceid=29, filesize=5.5 K
2024-11-17T12:49:55,628 DEBUG [M:0;0d751a622496:39839 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/98b181ca94414f10a0f9050422000992 as hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/98b181ca94414f10a0f9050422000992
2024-11-17T12:49:55,633 INFO  [M:0;0d751a622496:39839 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/98b181ca94414f10a0f9050422000992, entries=3, sequenceid=29, filesize=5.2 K
2024-11-17T12:49:55,634 DEBUG [M:0;0d751a622496:39839 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ec5922d6f26b4b3ea02e3dcbe37ec3f9 as hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ec5922d6f26b4b3ea02e3dcbe37ec3f9
2024-11-17T12:49:55,639 INFO  [M:0;0d751a622496:39839 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ec5922d6f26b4b3ea02e3dcbe37ec3f9, entries=1, sequenceid=29, filesize=5.0 K
2024-11-17T12:49:55,640 DEBUG [M:0;0d751a622496:39839 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/fba3bf304b4b4c49bc61cff34d640c29 as hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/fba3bf304b4b4c49bc61cff34d640c29
2024-11-17T12:49:55,644 INFO  [M:0;0d751a622496:39839 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40567/user/jenkins/test-data/857b059b-a8a1-254c-e56b-c6917a001653/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/fba3bf304b4b4c49bc61cff34d640c29, entries=1, sequenceid=29, filesize=4.9 K
2024-11-17T12:49:55,645 INFO  [M:0;0d751a622496:39839 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 135ms, sequenceid=29, compaction requested=false
2024-11-17T12:49:55,646 INFO  [M:0;0d751a622496:39839 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-17T12:49:55,647 DEBUG [M:0;0d751a622496:39839 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731847795510Disabling compacts and flushes for region at 1731847795510Disabling writes for close at 1731847795510Obtaining lock to block concurrent updates at 1731847795510Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731847795510Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1731847795511 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731847795511Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731847795512 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731847795527 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731847795527Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731847795543 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731847795564 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731847795564Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731847795573 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731847795592 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731847795592Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731847795600 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731847795614 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731847795614Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@542b95: reopening flushed file at 1731847795622 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30837d8d: reopening flushed file at 1731847795628 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e845b74: reopening flushed file at 1731847795633 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e3b5406: reopening flushed file at 1731847795639 (+6 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 135ms, sequenceid=29, compaction requested=false at 1731847795645 (+6 ms)Writing region close event to WAL at 1731847795646 (+1 ms)Closed at 1731847795646
2024-11-17T12:49:55,647 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:55,647 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:55,647 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:55,647 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:55,647 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-17T12:49:55,649 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35225 is added to blk_1073741830_1006 (size=10311)
2024-11-17T12:49:55,649 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45341 is added to blk_1073741830_1006 (size=10311)
2024-11-17T12:49:55,650 INFO  [M:0;0d751a622496:39839 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-17T12:49:55,650 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-17T12:49:55,650 INFO  [M:0;0d751a622496:39839 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39839
2024-11-17T12:49:55,650 INFO  [M:0;0d751a622496:39839 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-17T12:49:55,768 INFO  [M:0;0d751a622496:39839 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-17T12:49:55,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:49:55,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39839-0x1014a21bf980000, quorum=127.0.0.1:65257, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-17T12:49:55,770 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@76be211{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:49:55,770 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c2fc043{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:49:55,770 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:49:55,770 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@14628127{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:49:55,771 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@22ed154c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/hadoop.log.dir/,STOPPED}
2024-11-17T12:49:55,772 WARN  [BP-447412456-172.17.0.2-1731847792369 heartbeating to localhost/127.0.0.1:40567 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T12:49:55,772 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T12:49:55,772 WARN  [BP-447412456-172.17.0.2-1731847792369 heartbeating to localhost/127.0.0.1:40567 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-447412456-172.17.0.2-1731847792369 (Datanode Uuid 115ad31e-6643-4e06-b334-37e5800d61bf) service to localhost/127.0.0.1:40567
2024-11-17T12:49:55,772 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T12:49:55,772 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/cluster_f4408350-8688-a6d1-a2d2-82f46e2d86c0/data/data3/current/BP-447412456-172.17.0.2-1731847792369 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:49:55,772 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/cluster_f4408350-8688-a6d1-a2d2-82f46e2d86c0/data/data4/current/BP-447412456-172.17.0.2-1731847792369 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:49:55,772 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T12:49:55,775 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@23e03366{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-17T12:49:55,775 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2fb7753b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:49:55,775 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:49:55,775 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4bb82bb7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:49:55,775 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a1af98b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/hadoop.log.dir/,STOPPED}
2024-11-17T12:49:55,776 WARN  [BP-447412456-172.17.0.2-1731847792369 heartbeating to localhost/127.0.0.1:40567 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-17T12:49:55,776 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-17T12:49:55,777 WARN  [BP-447412456-172.17.0.2-1731847792369 heartbeating to localhost/127.0.0.1:40567 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-447412456-172.17.0.2-1731847792369 (Datanode Uuid 32ba399b-cbb6-4348-8c0b-8d94cf1df6a1) service to localhost/127.0.0.1:40567
2024-11-17T12:49:55,777 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-17T12:49:55,777 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/cluster_f4408350-8688-a6d1-a2d2-82f46e2d86c0/data/data1/current/BP-447412456-172.17.0.2-1731847792369 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:49:55,777 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/cluster_f4408350-8688-a6d1-a2d2-82f46e2d86c0/data/data2/current/BP-447412456-172.17.0.2-1731847792369 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-17T12:49:55,777 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-17T12:49:55,783 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6148e76b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-17T12:49:55,784 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3b55cb8{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-17T12:49:55,784 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-17T12:49:55,784 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3c703483{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-17T12:49:55,784 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7d664f93{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44d485d-b7f3-711b-05d9-7d24ce6d962c/hadoop.log.dir/,STOPPED}
2024-11-17T12:49:55,789 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-17T12:49:55,804 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-17T12:49:55,812 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=268 (was 229)
Potentially hanging thread: nioEventLoopGroup-43-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40567
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40567
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40567 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-16-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40567 from jenkins.hfs.7
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-16-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/0d751a622496:0.leaseChecker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40567
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:40567
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:40567
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40567 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)
 - Thread LEAK? -, OpenFileDescriptor=538 (was 515) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=226 (was 219) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4990 (was 4998)