2024-12-02 09:16:44,248 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca 2024-12-02 09:16:44,262 main DEBUG Took 0.011638 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-02 09:16:44,262 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-02 09:16:44,263 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-02 09:16:44,264 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-02 09:16:44,265 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 09:16:44,272 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-02 09:16:44,285 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 09:16:44,286 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 09:16:44,287 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 09:16:44,287 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 09:16:44,288 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 09:16:44,288 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 09:16:44,289 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 09:16:44,290 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 09:16:44,290 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 09:16:44,291 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 09:16:44,292 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 09:16:44,292 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 09:16:44,292 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 09:16:44,293 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 09:16:44,293 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 09:16:44,294 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 09:16:44,294 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 09:16:44,295 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 09:16:44,295 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 09:16:44,295 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 09:16:44,296 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 09:16:44,296 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 09:16:44,297 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 09:16:44,297 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 09:16:44,298 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 09:16:44,298 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-02 09:16:44,300 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 09:16:44,301 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-02 09:16:44,303 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-02 09:16:44,304 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-02 09:16:44,305 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-02 09:16:44,305 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-02 09:16:44,313 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-02 09:16:44,315 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-02 09:16:44,317 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-02 09:16:44,318 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-02 09:16:44,318 main DEBUG createAppenders(={Console}) 2024-12-02 09:16:44,319 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized 2024-12-02 09:16:44,320 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca 2024-12-02 09:16:44,320 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK. 2024-12-02 09:16:44,320 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-02 09:16:44,321 main DEBUG OutputStream closed 2024-12-02 09:16:44,321 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-02 09:16:44,321 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-02 09:16:44,322 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK 2024-12-02 09:16:44,387 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-02 09:16:44,390 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-02 09:16:44,391 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-02 09:16:44,392 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-02 09:16:44,392 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-02 09:16:44,393 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-02 09:16:44,393 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-02 09:16:44,394 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-02 09:16:44,394 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-02 09:16:44,394 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-02 09:16:44,395 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-02 09:16:44,395 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-02 09:16:44,395 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-02 09:16:44,396 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-02 09:16:44,396 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-02 09:16:44,396 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-02 09:16:44,396 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-02 09:16:44,397 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-02 09:16:44,399 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-02 09:16:44,399 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null 2024-12-02 09:16:44,400 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-02 09:16:44,400 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK. 2024-12-02T09:16:44,643 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b 2024-12-02 09:16:44,646 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-02 09:16:44,646 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-02T09:16:44,654 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins 2024-12-02T09:16:44,687 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=337, ProcessCount=11, AvailableMemoryMB=2102 2024-12-02T09:16:44,690 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-02T09:16:44,709 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/cluster_900e3e1e-6d1a-4bb5-1570-6ffe3d06df03, deleteOnExit=true 2024-12-02T09:16:44,709 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-02T09:16:44,710 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/test.cache.data in system properties and HBase conf 2024-12-02T09:16:44,711 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/hadoop.tmp.dir in system properties and HBase conf 2024-12-02T09:16:44,712 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/hadoop.log.dir in system properties and HBase conf 2024-12-02T09:16:44,712 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-02T09:16:44,713 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-02T09:16:44,713 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-02T09:16:44,810 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-02T09:16:44,903 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-02T09:16:44,908 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-02T09:16:44,908 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-02T09:16:44,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-02T09:16:44,910 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T09:16:44,910 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-02T09:16:44,911 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-02T09:16:44,911 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T09:16:44,912 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T09:16:44,913 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-02T09:16:44,913 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/nfs.dump.dir in system properties and HBase conf 2024-12-02T09:16:44,914 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/java.io.tmpdir in system properties and HBase conf 2024-12-02T09:16:44,914 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T09:16:44,915 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-02T09:16:44,915 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-02T09:16:45,375 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-02T09:16:45,822 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-02T09:16:45,893 INFO [Time-limited test {}] log.Log(170): Logging initialized @2373ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-02T09:16:45,971 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:16:46,045 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:16:46,078 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:16:46,078 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:16:46,080 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T09:16:46,099 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:16:46,102 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:16:46,103 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:16:46,277 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@735fa16a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/java.io.tmpdir/jetty-localhost-40401-hadoop-hdfs-3_4_1-tests_jar-_-any-15588265508506178166/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T09:16:46,286 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:40401} 2024-12-02T09:16:46,286 INFO [Time-limited test {}] server.Server(415): Started @2767ms 2024-12-02T09:16:46,314 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-02T09:16:46,795 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:16:46,803 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:16:46,808 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:16:46,808 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:16:46,808 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T09:16:46,809 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:16:46,810 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:16:46,951 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7b07d1ba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/java.io.tmpdir/jetty-localhost-39613-hadoop-hdfs-3_4_1-tests_jar-_-any-5685465935195311402/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:16:46,953 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:39613} 2024-12-02T09:16:46,953 INFO [Time-limited test {}] server.Server(415): Started @3434ms 2024-12-02T09:16:47,004 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T09:16:47,121 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:16:47,129 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:16:47,130 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:16:47,130 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:16:47,131 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T09:16:47,132 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:16:47,133 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:16:47,273 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bf97579{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/java.io.tmpdir/jetty-localhost-38091-hadoop-hdfs-3_4_1-tests_jar-_-any-156799259448975682/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:16:47,274 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:38091} 2024-12-02T09:16:47,274 INFO [Time-limited test {}] server.Server(415): Started @3755ms 2024-12-02T09:16:47,276 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T09:16:48,425 WARN [Thread-101 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/cluster_900e3e1e-6d1a-4bb5-1570-6ffe3d06df03/data/data4/current/BP-986440887-172.17.0.3-1733131005458/current, will proceed with Du for space computation calculation, 2024-12-02T09:16:48,425 WARN [Thread-99 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/cluster_900e3e1e-6d1a-4bb5-1570-6ffe3d06df03/data/data2/current/BP-986440887-172.17.0.3-1733131005458/current, will proceed with Du for space computation calculation, 2024-12-02T09:16:48,425 WARN [Thread-100 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/cluster_900e3e1e-6d1a-4bb5-1570-6ffe3d06df03/data/data3/current/BP-986440887-172.17.0.3-1733131005458/current, will proceed with Du for space computation calculation, 2024-12-02T09:16:48,425 WARN [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/cluster_900e3e1e-6d1a-4bb5-1570-6ffe3d06df03/data/data1/current/BP-986440887-172.17.0.3-1733131005458/current, will proceed with Du for space computation calculation, 2024-12-02T09:16:48,463 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T09:16:48,463 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T09:16:48,510 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8900309286432e22 with lease ID 0x93337d4d6868f1c2: Processing first storage report for DS-57938881-b63d-4ce8-abdb-68cff7369150 from datanode DatanodeRegistration(127.0.0.1:42095, datanodeUuid=bf87d23c-05b8-41cf-922c-097f536fca77, infoPort=38527, infoSecurePort=0, ipcPort=34225, storageInfo=lv=-57;cid=testClusterID;nsid=998026318;c=1733131005458) 2024-12-02T09:16:48,511 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8900309286432e22 with lease ID 0x93337d4d6868f1c2: from storage DS-57938881-b63d-4ce8-abdb-68cff7369150 node DatanodeRegistration(127.0.0.1:42095, datanodeUuid=bf87d23c-05b8-41cf-922c-097f536fca77, infoPort=38527, infoSecurePort=0, ipcPort=34225, storageInfo=lv=-57;cid=testClusterID;nsid=998026318;c=1733131005458), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-02T09:16:48,511 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x43ef35c73617d929 with lease ID 0x93337d4d6868f1c3: Processing first storage report for DS-549e9a62-8573-4a21-80cf-c6eb88542325 from datanode DatanodeRegistration(127.0.0.1:43645, datanodeUuid=6752efd9-c80c-4b0b-b968-4b06f5453def, infoPort=43595, infoSecurePort=0, ipcPort=36297, storageInfo=lv=-57;cid=testClusterID;nsid=998026318;c=1733131005458) 2024-12-02T09:16:48,512 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x43ef35c73617d929 with lease ID 0x93337d4d6868f1c3: from storage DS-549e9a62-8573-4a21-80cf-c6eb88542325 node DatanodeRegistration(127.0.0.1:43645, datanodeUuid=6752efd9-c80c-4b0b-b968-4b06f5453def, infoPort=43595, infoSecurePort=0, ipcPort=36297, storageInfo=lv=-57;cid=testClusterID;nsid=998026318;c=1733131005458), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:16:48,512 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8900309286432e22 with lease ID 0x93337d4d6868f1c2: Processing first storage report for DS-901de25f-5ac5-480b-9756-512346cb05d1 from datanode DatanodeRegistration(127.0.0.1:42095, datanodeUuid=bf87d23c-05b8-41cf-922c-097f536fca77, infoPort=38527, infoSecurePort=0, ipcPort=34225, storageInfo=lv=-57;cid=testClusterID;nsid=998026318;c=1733131005458) 2024-12-02T09:16:48,512 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8900309286432e22 with lease ID 0x93337d4d6868f1c2: from storage DS-901de25f-5ac5-480b-9756-512346cb05d1 node DatanodeRegistration(127.0.0.1:42095, datanodeUuid=bf87d23c-05b8-41cf-922c-097f536fca77, infoPort=38527, infoSecurePort=0, ipcPort=34225, storageInfo=lv=-57;cid=testClusterID;nsid=998026318;c=1733131005458), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:16:48,512 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x43ef35c73617d929 with lease ID 0x93337d4d6868f1c3: Processing first storage report for DS-216f1b36-2195-4174-8ca2-c74adad3cb74 from datanode DatanodeRegistration(127.0.0.1:43645, datanodeUuid=6752efd9-c80c-4b0b-b968-4b06f5453def, infoPort=43595, infoSecurePort=0, ipcPort=36297, storageInfo=lv=-57;cid=testClusterID;nsid=998026318;c=1733131005458) 2024-12-02T09:16:48,513 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x43ef35c73617d929 with lease ID 0x93337d4d6868f1c3: from storage DS-216f1b36-2195-4174-8ca2-c74adad3cb74 node DatanodeRegistration(127.0.0.1:43645, datanodeUuid=6752efd9-c80c-4b0b-b968-4b06f5453def, infoPort=43595, infoSecurePort=0, ipcPort=36297, storageInfo=lv=-57;cid=testClusterID;nsid=998026318;c=1733131005458), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:16:48,587 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b 2024-12-02T09:16:48,649 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/cluster_900e3e1e-6d1a-4bb5-1570-6ffe3d06df03/zookeeper_0, clientPort=53155, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/cluster_900e3e1e-6d1a-4bb5-1570-6ffe3d06df03/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/cluster_900e3e1e-6d1a-4bb5-1570-6ffe3d06df03/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-02T09:16:48,658 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53155 2024-12-02T09:16:48,669 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:16:48,672 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:16:48,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741825_1001 (size=7) 2024-12-02T09:16:48,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741825_1001 (size=7) 2024-12-02T09:16:49,314 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a with version=8 2024-12-02T09:16:49,315 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/hbase-staging 2024-12-02T09:16:49,396 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-02T09:16:49,596 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7c6d666a4939:0 server-side Connection retries=45 2024-12-02T09:16:49,605 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:16:49,606 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T09:16:49,610 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T09:16:49,610 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:16:49,610 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T09:16:49,779 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-02T09:16:49,837 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-02T09:16:49,845 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-02T09:16:49,849 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T09:16:49,874 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 47688 (auto-detected) 2024-12-02T09:16:49,875 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-02T09:16:49,895 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43095 2024-12-02T09:16:49,913 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43095 connecting to ZooKeeper ensemble=127.0.0.1:53155 2024-12-02T09:16:49,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:430950x0, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T09:16:49,985 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43095-0x1009a4659ad0000 connected 2024-12-02T09:16:50,058 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:16:50,060 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:16:50,068 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:16:50,072 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a, hbase.cluster.distributed=false 2024-12-02T09:16:50,098 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T09:16:50,104 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43095 2024-12-02T09:16:50,104 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43095 2024-12-02T09:16:50,105 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43095 2024-12-02T09:16:50,105 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43095 2024-12-02T09:16:50,106 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43095 2024-12-02T09:16:50,208 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7c6d666a4939:0 server-side Connection retries=45 2024-12-02T09:16:50,210 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:16:50,210 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T09:16:50,210 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T09:16:50,210 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:16:50,210 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T09:16:50,212 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T09:16:50,214 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T09:16:50,215 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38963 2024-12-02T09:16:50,217 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38963 connecting to ZooKeeper ensemble=127.0.0.1:53155 2024-12-02T09:16:50,218 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:16:50,221 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:16:50,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:389630x0, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T09:16:50,233 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:389630x0, quorum=127.0.0.1:53155, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:16:50,237 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T09:16:50,239 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38963-0x1009a4659ad0001 connected 2024-12-02T09:16:50,245 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T09:16:50,247 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38963-0x1009a4659ad0001, quorum=127.0.0.1:53155, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T09:16:50,252 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38963-0x1009a4659ad0001, quorum=127.0.0.1:53155, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T09:16:50,253 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38963 2024-12-02T09:16:50,254 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38963 2024-12-02T09:16:50,254 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38963 2024-12-02T09:16:50,255 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38963 2024-12-02T09:16:50,258 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38963 2024-12-02T09:16:50,275 DEBUG [M:0;7c6d666a4939:43095 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7c6d666a4939:43095 2024-12-02T09:16:50,276 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7c6d666a4939,43095,1733131009439 2024-12-02T09:16:50,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38963-0x1009a4659ad0001, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:16:50,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:16:50,294 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7c6d666a4939,43095,1733131009439 2024-12-02T09:16:50,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38963-0x1009a4659ad0001, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T09:16:50,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:16:50,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38963-0x1009a4659ad0001, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:16:50,325 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-02T09:16:50,326 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7c6d666a4939,43095,1733131009439 from backup master directory 2024-12-02T09:16:50,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7c6d666a4939,43095,1733131009439 2024-12-02T09:16:50,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38963-0x1009a4659ad0001, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:16:50,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:16:50,339 WARN [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T09:16:50,339 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7c6d666a4939,43095,1733131009439 2024-12-02T09:16:50,342 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-02T09:16:50,344 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-02T09:16:50,390 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/hbase.id] with ID: f0859d69-9dd0-4144-a385-1bbe2c5f9bf6 2024-12-02T09:16:50,390 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/.tmp/hbase.id 2024-12-02T09:16:50,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741826_1002 (size=42) 2024-12-02T09:16:50,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741826_1002 (size=42) 2024-12-02T09:16:50,402 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/.tmp/hbase.id]:[hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/hbase.id] 2024-12-02T09:16:50,443 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:16:50,447 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-02T09:16:50,463 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 14ms. 2024-12-02T09:16:50,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:16:50,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38963-0x1009a4659ad0001, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:16:50,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741827_1003 (size=196) 2024-12-02T09:16:50,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741827_1003 (size=196) 2024-12-02T09:16:50,505 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T09:16:50,507 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-02T09:16:50,511 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-02T09:16:50,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741828_1004 (size=1189) 2024-12-02T09:16:50,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741828_1004 (size=1189) 2024-12-02T09:16:50,554 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/data/master/store 2024-12-02T09:16:50,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741829_1005 (size=34) 2024-12-02T09:16:50,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741829_1005 (size=34) 2024-12-02T09:16:50,576 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-02T09:16:50,579 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:16:50,580 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T09:16:50,580 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:16:50,580 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:16:50,581 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T09:16:50,582 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:16:50,582 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:16:50,583 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733131010580Disabling compacts and flushes for region at 1733131010580Disabling writes for close at 1733131010581 (+1 ms)Writing region close event to WAL at 1733131010582 (+1 ms)Closed at 1733131010582 2024-12-02T09:16:50,585 WARN [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/data/master/store/.initializing 2024-12-02T09:16:50,585 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/WALs/7c6d666a4939,43095,1733131009439 2024-12-02T09:16:50,606 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c6d666a4939%2C43095%2C1733131009439, suffix=, logDir=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/WALs/7c6d666a4939,43095,1733131009439, archiveDir=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/oldWALs, maxLogs=10 2024-12-02T09:16:50,614 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C43095%2C1733131009439.1733131010610 2024-12-02T09:16:50,637 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/WALs/7c6d666a4939,43095,1733131009439/7c6d666a4939%2C43095%2C1733131009439.1733131010610 2024-12-02T09:16:50,645 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43595:43595),(127.0.0.1/127.0.0.1:38527:38527)] 2024-12-02T09:16:50,646 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-02T09:16:50,646 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:16:50,649 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:16:50,650 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:16:50,683 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:16:50,710 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-02T09:16:50,714 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:16:50,716 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:16:50,717 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:16:50,720 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-02T09:16:50,721 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:16:50,722 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:16:50,723 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:16:50,725 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-02T09:16:50,726 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:16:50,727 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:16:50,727 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:16:50,729 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-02T09:16:50,730 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:16:50,731 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:16:50,731 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:16:50,735 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:16:50,736 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:16:50,741 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:16:50,742 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:16:50,745 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T09:16:50,748 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:16:50,752 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T09:16:50,754 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=805692, jitterRate=0.02449069917201996}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T09:16:50,759 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733131010662Initializing all the Stores at 1733131010664 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131010664Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131010665 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131010665Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131010665Cleaning up temporary data from old regions at 1733131010742 (+77 ms)Region opened successfully at 1733131010759 (+17 ms) 2024-12-02T09:16:50,760 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-02T09:16:50,797 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f94f83c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7c6d666a4939/172.17.0.3:0 2024-12-02T09:16:50,836 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-02T09:16:50,847 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-02T09:16:50,847 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-02T09:16:50,850 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T09:16:50,852 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-02T09:16:50,856 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-02T09:16:50,856 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-02T09:16:50,881 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-02T09:16:50,888 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-02T09:16:50,899 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-02T09:16:50,902 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-02T09:16:50,903 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-02T09:16:50,913 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-02T09:16:50,915 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-02T09:16:50,918 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-02T09:16:50,924 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-02T09:16:50,925 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-02T09:16:50,932 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-02T09:16:50,948 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-02T09:16:50,957 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-02T09:16:50,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T09:16:50,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38963-0x1009a4659ad0001, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T09:16:50,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:16:50,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38963-0x1009a4659ad0001, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:16:50,968 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7c6d666a4939,43095,1733131009439, sessionid=0x1009a4659ad0000, setting cluster-up flag (Was=false) 2024-12-02T09:16:50,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38963-0x1009a4659ad0001, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:16:50,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:16:51,015 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-02T09:16:51,018 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7c6d666a4939,43095,1733131009439 2024-12-02T09:16:51,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:16:51,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38963-0x1009a4659ad0001, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:16:51,065 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-02T09:16:51,067 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7c6d666a4939,43095,1733131009439 2024-12-02T09:16:51,074 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-02T09:16:51,136 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T09:16:51,145 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-02T09:16:51,152 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-02T09:16:51,159 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7c6d666a4939,43095,1733131009439 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-02T09:16:51,164 INFO [RS:0;7c6d666a4939:38963 {}] regionserver.HRegionServer(746): ClusterId : f0859d69-9dd0-4144-a385-1bbe2c5f9bf6 2024-12-02T09:16:51,166 DEBUG [RS:0;7c6d666a4939:38963 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T09:16:51,168 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:16:51,169 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:16:51,169 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:16:51,169 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:16:51,169 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7c6d666a4939:0, corePoolSize=10, maxPoolSize=10 2024-12-02T09:16:51,169 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:16:51,169 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7c6d666a4939:0, corePoolSize=2, maxPoolSize=2 2024-12-02T09:16:51,170 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:16:51,171 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733131041171 2024-12-02T09:16:51,173 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-02T09:16:51,174 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-02T09:16:51,176 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T09:16:51,177 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-02T09:16:51,179 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-02T09:16:51,179 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-02T09:16:51,180 DEBUG [RS:0;7c6d666a4939:38963 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T09:16:51,180 DEBUG [RS:0;7c6d666a4939:38963 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T09:16:51,180 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-02T09:16:51,180 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-02T09:16:51,181 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T09:16:51,183 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:16:51,184 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-02T09:16:51,184 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-02T09:16:51,186 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-02T09:16:51,187 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-02T09:16:51,189 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-02T09:16:51,189 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-02T09:16:51,191 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.large.0-1733131011190,5,FailOnTimeoutGroup] 2024-12-02T09:16:51,192 DEBUG [RS:0;7c6d666a4939:38963 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T09:16:51,193 DEBUG [RS:0;7c6d666a4939:38963 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c5c1cdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7c6d666a4939/172.17.0.3:0 2024-12-02T09:16:51,195 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.small.0-1733131011192,5,FailOnTimeoutGroup] 2024-12-02T09:16:51,195 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T09:16:51,196 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-02T09:16:51,197 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-02T09:16:51,198 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-02T09:16:51,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741831_1007 (size=1321) 2024-12-02T09:16:51,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741831_1007 (size=1321) 2024-12-02T09:16:51,206 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-02T09:16:51,207 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a 2024-12-02T09:16:51,215 DEBUG [RS:0;7c6d666a4939:38963 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7c6d666a4939:38963 2024-12-02T09:16:51,218 INFO [RS:0;7c6d666a4939:38963 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T09:16:51,219 INFO [RS:0;7c6d666a4939:38963 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T09:16:51,219 DEBUG [RS:0;7c6d666a4939:38963 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T09:16:51,221 INFO [RS:0;7c6d666a4939:38963 {}] regionserver.HRegionServer(2659): reportForDuty to master=7c6d666a4939,43095,1733131009439 with port=38963, startcode=1733131010177 2024-12-02T09:16:51,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741832_1008 (size=32) 2024-12-02T09:16:51,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741832_1008 (size=32) 2024-12-02T09:16:51,227 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:16:51,230 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T09:16:51,233 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T09:16:51,233 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:16:51,234 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:16:51,234 DEBUG [RS:0;7c6d666a4939:38963 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T09:16:51,234 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T09:16:51,236 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T09:16:51,236 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:16:51,237 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:16:51,238 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T09:16:51,241 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T09:16:51,241 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:16:51,242 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:16:51,242 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T09:16:51,245 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T09:16:51,245 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:16:51,246 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:16:51,246 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T09:16:51,248 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/hbase/meta/1588230740 2024-12-02T09:16:51,249 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/hbase/meta/1588230740 2024-12-02T09:16:51,252 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T09:16:51,252 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T09:16:51,253 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T09:16:51,256 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T09:16:51,260 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T09:16:51,261 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=740354, jitterRate=-0.0585920512676239}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T09:16:51,264 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733131011228Initializing all the Stores at 1733131011230 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131011230Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131011230Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131011230Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131011230Cleaning up temporary data from old regions at 1733131011252 (+22 ms)Region opened successfully at 1733131011264 (+12 ms) 2024-12-02T09:16:51,265 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T09:16:51,265 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T09:16:51,265 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T09:16:51,265 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T09:16:51,265 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T09:16:51,266 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T09:16:51,267 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733131011264Disabling compacts and flushes for region at 1733131011265 (+1 ms)Disabling writes for close at 1733131011265Writing region close event to WAL at 1733131011266 (+1 ms)Closed at 1733131011266 2024-12-02T09:16:51,270 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T09:16:51,271 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-02T09:16:51,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-02T09:16:51,290 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T09:16:51,294 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-02T09:16:51,307 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59063, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T09:16:51,313 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43095 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7c6d666a4939,38963,1733131010177 2024-12-02T09:16:51,315 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43095 {}] master.ServerManager(517): Registering regionserver=7c6d666a4939,38963,1733131010177 2024-12-02T09:16:51,327 DEBUG [RS:0;7c6d666a4939:38963 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a 2024-12-02T09:16:51,327 DEBUG [RS:0;7c6d666a4939:38963 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37857 2024-12-02T09:16:51,327 DEBUG [RS:0;7c6d666a4939:38963 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T09:16:51,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T09:16:51,341 DEBUG [RS:0;7c6d666a4939:38963 {}] zookeeper.ZKUtil(111): regionserver:38963-0x1009a4659ad0001, quorum=127.0.0.1:53155, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7c6d666a4939,38963,1733131010177 2024-12-02T09:16:51,342 WARN [RS:0;7c6d666a4939:38963 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T09:16:51,342 INFO [RS:0;7c6d666a4939:38963 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-02T09:16:51,342 DEBUG [RS:0;7c6d666a4939:38963 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177 2024-12-02T09:16:51,343 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7c6d666a4939,38963,1733131010177] 2024-12-02T09:16:51,368 INFO [RS:0;7c6d666a4939:38963 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T09:16:51,380 INFO [RS:0;7c6d666a4939:38963 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T09:16:51,384 INFO [RS:0;7c6d666a4939:38963 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T09:16:51,385 INFO [RS:0;7c6d666a4939:38963 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:16:51,386 INFO [RS:0;7c6d666a4939:38963 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T09:16:51,393 INFO [RS:0;7c6d666a4939:38963 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T09:16:51,394 INFO [RS:0;7c6d666a4939:38963 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T09:16:51,394 DEBUG [RS:0;7c6d666a4939:38963 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:16:51,394 DEBUG [RS:0;7c6d666a4939:38963 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:16:51,394 DEBUG [RS:0;7c6d666a4939:38963 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:16:51,394 DEBUG [RS:0;7c6d666a4939:38963 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:16:51,395 DEBUG [RS:0;7c6d666a4939:38963 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:16:51,395 DEBUG [RS:0;7c6d666a4939:38963 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7c6d666a4939:0, corePoolSize=2, maxPoolSize=2 2024-12-02T09:16:51,395 DEBUG [RS:0;7c6d666a4939:38963 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:16:51,395 DEBUG [RS:0;7c6d666a4939:38963 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:16:51,395 DEBUG [RS:0;7c6d666a4939:38963 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:16:51,395 DEBUG [RS:0;7c6d666a4939:38963 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:16:51,395 DEBUG [RS:0;7c6d666a4939:38963 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:16:51,395 DEBUG [RS:0;7c6d666a4939:38963 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:16:51,396 DEBUG [RS:0;7c6d666a4939:38963 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7c6d666a4939:0, corePoolSize=3, maxPoolSize=3 2024-12-02T09:16:51,396 DEBUG [RS:0;7c6d666a4939:38963 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0, corePoolSize=3, maxPoolSize=3 2024-12-02T09:16:51,397 INFO [RS:0;7c6d666a4939:38963 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T09:16:51,397 INFO [RS:0;7c6d666a4939:38963 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T09:16:51,397 INFO [RS:0;7c6d666a4939:38963 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:16:51,397 INFO [RS:0;7c6d666a4939:38963 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T09:16:51,397 INFO [RS:0;7c6d666a4939:38963 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T09:16:51,397 INFO [RS:0;7c6d666a4939:38963 {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,38963,1733131010177-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T09:16:51,416 INFO [RS:0;7c6d666a4939:38963 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T09:16:51,418 INFO [RS:0;7c6d666a4939:38963 {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,38963,1733131010177-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:16:51,418 INFO [RS:0;7c6d666a4939:38963 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:16:51,418 INFO [RS:0;7c6d666a4939:38963 {}] regionserver.Replication(171): 7c6d666a4939,38963,1733131010177 started 2024-12-02T09:16:51,434 INFO [RS:0;7c6d666a4939:38963 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:16:51,434 INFO [RS:0;7c6d666a4939:38963 {}] regionserver.HRegionServer(1482): Serving as 7c6d666a4939,38963,1733131010177, RpcServer on 7c6d666a4939/172.17.0.3:38963, sessionid=0x1009a4659ad0001 2024-12-02T09:16:51,435 DEBUG [RS:0;7c6d666a4939:38963 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T09:16:51,435 DEBUG [RS:0;7c6d666a4939:38963 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7c6d666a4939,38963,1733131010177 2024-12-02T09:16:51,436 DEBUG [RS:0;7c6d666a4939:38963 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7c6d666a4939,38963,1733131010177' 2024-12-02T09:16:51,436 DEBUG [RS:0;7c6d666a4939:38963 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T09:16:51,437 DEBUG [RS:0;7c6d666a4939:38963 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T09:16:51,437 DEBUG [RS:0;7c6d666a4939:38963 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T09:16:51,437 DEBUG [RS:0;7c6d666a4939:38963 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T09:16:51,437 DEBUG [RS:0;7c6d666a4939:38963 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7c6d666a4939,38963,1733131010177 2024-12-02T09:16:51,438 DEBUG [RS:0;7c6d666a4939:38963 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7c6d666a4939,38963,1733131010177' 2024-12-02T09:16:51,438 DEBUG [RS:0;7c6d666a4939:38963 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T09:16:51,438 DEBUG [RS:0;7c6d666a4939:38963 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T09:16:51,439 DEBUG [RS:0;7c6d666a4939:38963 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T09:16:51,439 INFO [RS:0;7c6d666a4939:38963 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T09:16:51,439 INFO [RS:0;7c6d666a4939:38963 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T09:16:51,445 WARN [7c6d666a4939:43095 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-02T09:16:51,546 INFO [RS:0;7c6d666a4939:38963 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c6d666a4939%2C38963%2C1733131010177, suffix=, logDir=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177, archiveDir=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/oldWALs, maxLogs=32 2024-12-02T09:16:51,550 INFO [RS:0;7c6d666a4939:38963 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C38963%2C1733131010177.1733131011549 2024-12-02T09:16:51,559 INFO [RS:0;7c6d666a4939:38963 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131011549 2024-12-02T09:16:51,561 DEBUG [RS:0;7c6d666a4939:38963 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43595:43595),(127.0.0.1/127.0.0.1:38527:38527)] 2024-12-02T09:16:51,699 DEBUG [7c6d666a4939:43095 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-02T09:16:51,714 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7c6d666a4939,38963,1733131010177 2024-12-02T09:16:51,719 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7c6d666a4939,38963,1733131010177, state=OPENING 2024-12-02T09:16:51,732 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-02T09:16:51,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38963-0x1009a4659ad0001, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:16:51,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:16:51,741 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:16:51,741 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:16:51,743 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T09:16:51,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7c6d666a4939,38963,1733131010177}] 2024-12-02T09:16:51,929 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T09:16:51,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53861, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T09:16:51,943 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-02T09:16:51,944 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-02T09:16:51,947 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c6d666a4939%2C38963%2C1733131010177.meta, suffix=.meta, logDir=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177, archiveDir=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/oldWALs, maxLogs=32 2024-12-02T09:16:51,949 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C38963%2C1733131010177.meta.1733131011949.meta 2024-12-02T09:16:51,957 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.meta.1733131011949.meta 2024-12-02T09:16:51,959 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43595:43595),(127.0.0.1/127.0.0.1:38527:38527)] 2024-12-02T09:16:51,963 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-02T09:16:51,966 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-02T09:16:51,968 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-02T09:16:51,972 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-02T09:16:51,976 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-02T09:16:51,977 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:16:51,977 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-02T09:16:51,977 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-02T09:16:51,980 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T09:16:51,982 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T09:16:51,982 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:16:51,983 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:16:51,983 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T09:16:51,985 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T09:16:51,985 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:16:51,986 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:16:51,986 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T09:16:51,988 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T09:16:51,988 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:16:51,989 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:16:51,989 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T09:16:51,991 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T09:16:51,991 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:16:51,992 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:16:51,992 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T09:16:51,994 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/hbase/meta/1588230740 2024-12-02T09:16:51,997 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/hbase/meta/1588230740 2024-12-02T09:16:51,999 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T09:16:51,999 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T09:16:52,000 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T09:16:52,002 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T09:16:52,004 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=807773, jitterRate=0.027137428522109985}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T09:16:52,004 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-02T09:16:52,005 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733131011978Writing region info on filesystem at 1733131011978Initializing all the Stores at 1733131011979 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131011980 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131011980Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131011980Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131011980Cleaning up temporary data from old regions at 1733131011999 (+19 ms)Running coprocessor post-open hooks at 1733131012004 (+5 ms)Region opened successfully at 1733131012005 (+1 ms) 2024-12-02T09:16:52,012 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733131011921 2024-12-02T09:16:52,022 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-02T09:16:52,023 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-02T09:16:52,024 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7c6d666a4939,38963,1733131010177 2024-12-02T09:16:52,026 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7c6d666a4939,38963,1733131010177, state=OPEN 2024-12-02T09:16:52,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38963-0x1009a4659ad0001, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T09:16:52,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T09:16:52,061 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:16:52,061 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:16:52,061 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7c6d666a4939,38963,1733131010177 2024-12-02T09:16:52,067 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-02T09:16:52,068 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7c6d666a4939,38963,1733131010177 in 315 msec 2024-12-02T09:16:52,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-02T09:16:52,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 791 msec 2024-12-02T09:16:52,077 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T09:16:52,077 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-02T09:16:52,098 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T09:16:52,099 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c6d666a4939,38963,1733131010177, seqNum=-1] 2024-12-02T09:16:52,118 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T09:16:52,120 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45571, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T09:16:52,140 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0370 sec 2024-12-02T09:16:52,141 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733131012141, completionTime=-1 2024-12-02T09:16:52,144 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-02T09:16:52,144 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T09:16:52,168 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-02T09:16:52,168 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733131072168 2024-12-02T09:16:52,168 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733131132168 2024-12-02T09:16:52,168 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-12-02T09:16:52,171 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,43095,1733131009439-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:16:52,171 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,43095,1733131009439-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:16:52,171 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,43095,1733131009439-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:16:52,173 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7c6d666a4939:43095, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:16:52,173 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-02T09:16:52,174 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-02T09:16:52,179 DEBUG [master/7c6d666a4939:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-02T09:16:52,198 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.858sec 2024-12-02T09:16:52,199 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-02T09:16:52,200 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-02T09:16:52,201 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-02T09:16:52,202 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-02T09:16:52,202 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-02T09:16:52,202 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,43095,1733131009439-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T09:16:52,203 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,43095,1733131009439-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-02T09:16:52,211 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-02T09:16:52,212 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-02T09:16:52,213 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,43095,1733131009439-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:16:52,279 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@358a6cb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T09:16:52,281 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-02T09:16:52,282 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-02T09:16:52,285 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c6d666a4939,43095,-1 for getting cluster id 2024-12-02T09:16:52,288 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T09:16:52,295 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f0859d69-9dd0-4144-a385-1bbe2c5f9bf6' 2024-12-02T09:16:52,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T09:16:52,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f0859d69-9dd0-4144-a385-1bbe2c5f9bf6" 2024-12-02T09:16:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29842d21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T09:16:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c6d666a4939,43095,-1] 2024-12-02T09:16:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T09:16:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:16:52,305 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T09:16:52,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19f64e80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T09:16:52,308 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T09:16:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c6d666a4939,38963,1733131010177, seqNum=-1] 2024-12-02T09:16:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T09:16:52,317 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T09:16:52,336 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7c6d666a4939,43095,1733131009439 2024-12-02T09:16:52,336 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:16:52,343 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-12-02T09:16:52,347 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T09:16:52,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 7c6d666a4939,43095,1733131009439 2024-12-02T09:16:52,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2bfd4db8 2024-12-02T09:16:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T09:16:52,356 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T09:16:52,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-02T09:16:52,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-02T09:16:52,361 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T09:16:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling 2024-12-02T09:16:52,371 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T09:16:52,373 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testSlowSyncLogRolling" procId is: 4 2024-12-02T09:16:52,374 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:16:52,377 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T09:16:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T09:16:52,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741835_1011 (size=389) 2024-12-02T09:16:52,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741835_1011 (size=389) 2024-12-02T09:16:52,424 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d702b350e36b4ab2fde0db91f63c1c5d, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a 2024-12-02T09:16:52,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741836_1012 (size=72) 2024-12-02T09:16:52,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741836_1012 (size=72) 2024-12-02T09:16:52,446 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:16:52,446 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing d702b350e36b4ab2fde0db91f63c1c5d, disabling compactions & flushes 2024-12-02T09:16:52,446 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d. 2024-12-02T09:16:52,446 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d. 2024-12-02T09:16:52,446 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d. after waiting 0 ms 2024-12-02T09:16:52,446 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d. 2024-12-02T09:16:52,446 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d. 2024-12-02T09:16:52,446 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for d702b350e36b4ab2fde0db91f63c1c5d: Waiting for close lock at 1733131012446Disabling compacts and flushes for region at 1733131012446Disabling writes for close at 1733131012446Writing region close event to WAL at 1733131012446Closed at 1733131012446 2024-12-02T09:16:52,449 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T09:16:52,455 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1733131012449"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733131012449"}]},"ts":"1733131012449"} 2024-12-02T09:16:52,461 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T09:16:52,463 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T09:16:52,466 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733131012463"}]},"ts":"1733131012463"} 2024-12-02T09:16:52,471 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta 2024-12-02T09:16:52,488 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=d702b350e36b4ab2fde0db91f63c1c5d, ASSIGN}] 2024-12-02T09:16:52,493 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=d702b350e36b4ab2fde0db91f63c1c5d, ASSIGN 2024-12-02T09:16:52,496 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=d702b350e36b4ab2fde0db91f63c1c5d, ASSIGN; state=OFFLINE, location=7c6d666a4939,38963,1733131010177; forceNewPlan=false, retain=false 2024-12-02T09:16:52,647 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d702b350e36b4ab2fde0db91f63c1c5d, regionState=OPENING, regionLocation=7c6d666a4939,38963,1733131010177 2024-12-02T09:16:52,652 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=d702b350e36b4ab2fde0db91f63c1c5d, ASSIGN because future has completed 2024-12-02T09:16:52,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d702b350e36b4ab2fde0db91f63c1c5d, server=7c6d666a4939,38963,1733131010177}] 2024-12-02T09:16:52,814 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d. 2024-12-02T09:16:52,814 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => d702b350e36b4ab2fde0db91f63c1c5d, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d.', STARTKEY => '', ENDKEY => ''} 2024-12-02T09:16:52,815 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling d702b350e36b4ab2fde0db91f63c1c5d 2024-12-02T09:16:52,815 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:16:52,816 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for d702b350e36b4ab2fde0db91f63c1c5d 2024-12-02T09:16:52,816 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for d702b350e36b4ab2fde0db91f63c1c5d 2024-12-02T09:16:52,818 INFO [StoreOpener-d702b350e36b4ab2fde0db91f63c1c5d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region d702b350e36b4ab2fde0db91f63c1c5d 2024-12-02T09:16:52,820 INFO [StoreOpener-d702b350e36b4ab2fde0db91f63c1c5d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d702b350e36b4ab2fde0db91f63c1c5d columnFamilyName info 2024-12-02T09:16:52,821 DEBUG [StoreOpener-d702b350e36b4ab2fde0db91f63c1c5d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:16:52,822 INFO [StoreOpener-d702b350e36b4ab2fde0db91f63c1c5d-1 {}] regionserver.HStore(327): Store=d702b350e36b4ab2fde0db91f63c1c5d/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:16:52,822 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for d702b350e36b4ab2fde0db91f63c1c5d 2024-12-02T09:16:52,823 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d 2024-12-02T09:16:52,824 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d 2024-12-02T09:16:52,825 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for d702b350e36b4ab2fde0db91f63c1c5d 2024-12-02T09:16:52,825 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for d702b350e36b4ab2fde0db91f63c1c5d 2024-12-02T09:16:52,829 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for d702b350e36b4ab2fde0db91f63c1c5d 2024-12-02T09:16:52,836 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T09:16:52,838 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened d702b350e36b4ab2fde0db91f63c1c5d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=751178, jitterRate=-0.04482893645763397}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T09:16:52,838 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d702b350e36b4ab2fde0db91f63c1c5d 2024-12-02T09:16:52,839 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for d702b350e36b4ab2fde0db91f63c1c5d: Running coprocessor pre-open hook at 1733131012816Writing region info on filesystem at 1733131012816Initializing all the Stores at 1733131012818 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131012818Cleaning up temporary data from old regions at 1733131012825 (+7 ms)Running coprocessor post-open hooks at 1733131012838 (+13 ms)Region opened successfully at 1733131012839 (+1 ms) 2024-12-02T09:16:52,841 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d., pid=6, masterSystemTime=1733131012808 2024-12-02T09:16:52,845 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d. 2024-12-02T09:16:52,845 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d. 2024-12-02T09:16:52,846 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d702b350e36b4ab2fde0db91f63c1c5d, regionState=OPEN, openSeqNum=2, regionLocation=7c6d666a4939,38963,1733131010177 2024-12-02T09:16:52,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d702b350e36b4ab2fde0db91f63c1c5d, server=7c6d666a4939,38963,1733131010177 because future has completed 2024-12-02T09:16:52,857 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-02T09:16:52,858 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d702b350e36b4ab2fde0db91f63c1c5d, server=7c6d666a4939,38963,1733131010177 in 199 msec 2024-12-02T09:16:52,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-02T09:16:52,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=d702b350e36b4ab2fde0db91f63c1c5d, ASSIGN in 369 msec 2024-12-02T09:16:52,864 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T09:16:52,864 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733131012864"}]},"ts":"1733131012864"} 2024-12-02T09:16:52,868 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta 2024-12-02T09:16:52,870 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T09:16:52,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 507 msec 2024-12-02T09:16:57,492 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T09:16:57,534 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-02T09:16:57,536 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling' 2024-12-02T09:16:59,835 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-02T09:16:59,836 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-02T09:16:59,841 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-12-02T09:16:59,841 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer 2024-12-02T09:16:59,843 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T09:16:59,843 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-02T09:16:59,844 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-02T09:16:59,844 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-02T09:17:02,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43095 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T09:17:02,436 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed 2024-12-02T09:17:02,440 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100 2024-12-02T09:17:02,448 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling 2024-12-02T09:17:02,448 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d. 2024-12-02T09:17:02,449 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C38963%2C1733131010177.1733131022449 2024-12-02T09:17:02,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:17:02,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:17:02,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:17:02,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:17:02,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:17:02,459 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131011549 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131022449 2024-12-02T09:17:02,460 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38527:38527),(127.0.0.1/127.0.0.1:43595:43595)] 2024-12-02T09:17:02,460 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131011549 is not closed yet, will try archiving it next time 2024-12-02T09:17:02,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741833_1009 (size=451) 2024-12-02T09:17:02,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741833_1009 (size=451) 2024-12-02T09:17:02,465 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131011549 to hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/oldWALs/7c6d666a4939%2C38963%2C1733131010177.1733131011549 2024-12-02T09:17:02,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d., hostname=7c6d666a4939,38963,1733131010177, seqNum=2] 2024-12-02T09:17:14,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38963 {}] regionserver.HRegion(8855): Flush requested on d702b350e36b4ab2fde0db91f63c1c5d 2024-12-02T09:17:14,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d702b350e36b4ab2fde0db91f63c1c5d 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-02T09:17:14,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/.tmp/info/a5f327fda3de443192e2ca39aaaa205a is 1080, key is row0001/info:/1733131022473/Put/seqid=0 2024-12-02T09:17:14,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741838_1014 (size=12509) 2024-12-02T09:17:14,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741838_1014 (size=12509) 2024-12-02T09:17:14,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/.tmp/info/a5f327fda3de443192e2ca39aaaa205a 2024-12-02T09:17:14,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/.tmp/info/a5f327fda3de443192e2ca39aaaa205a as hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/a5f327fda3de443192e2ca39aaaa205a 2024-12-02T09:17:14,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/a5f327fda3de443192e2ca39aaaa205a, entries=7, sequenceid=11, filesize=12.2 K 2024-12-02T09:17:14,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for d702b350e36b4ab2fde0db91f63c1c5d in 136ms, sequenceid=11, compaction requested=false 2024-12-02T09:17:14,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d702b350e36b4ab2fde0db91f63c1c5d: 2024-12-02T09:17:18,584 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T09:17:22,524 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C38963%2C1733131010177.1733131042524 2024-12-02T09:17:22,732 INFO [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42095,DS-57938881-b63d-4ce8-abdb-68cff7369150,DISK], DatanodeInfoWithStorage[127.0.0.1:43645,DS-549e9a62-8573-4a21-80cf-c6eb88542325,DISK]] 2024-12-02T09:17:22,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:17:22,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:17:22,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:17:22,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:17:22,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:17:22,733 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131022449 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131042524 2024-12-02T09:17:22,734 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43595:43595),(127.0.0.1/127.0.0.1:38527:38527)] 2024-12-02T09:17:22,734 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131022449 is not closed yet, will try archiving it next time 2024-12-02T09:17:22,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741837_1013 (size=12399) 2024-12-02T09:17:22,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741837_1013 (size=12399) 2024-12-02T09:17:22,938 INFO [FSHLog-0-hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a-prefix:7c6d666a4939,38963,1733131010177 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43645,DS-549e9a62-8573-4a21-80cf-c6eb88542325,DISK], DatanodeInfoWithStorage[127.0.0.1:42095,DS-57938881-b63d-4ce8-abdb-68cff7369150,DISK]] 2024-12-02T09:17:25,141 INFO [FSHLog-0-hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a-prefix:7c6d666a4939,38963,1733131010177 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43645,DS-549e9a62-8573-4a21-80cf-c6eb88542325,DISK], DatanodeInfoWithStorage[127.0.0.1:42095,DS-57938881-b63d-4ce8-abdb-68cff7369150,DISK]] 2024-12-02T09:17:27,345 INFO [FSHLog-0-hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a-prefix:7c6d666a4939,38963,1733131010177 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43645,DS-549e9a62-8573-4a21-80cf-c6eb88542325,DISK], DatanodeInfoWithStorage[127.0.0.1:42095,DS-57938881-b63d-4ce8-abdb-68cff7369150,DISK]] 2024-12-02T09:17:29,549 INFO [FSHLog-0-hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a-prefix:7c6d666a4939,38963,1733131010177 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43645,DS-549e9a62-8573-4a21-80cf-c6eb88542325,DISK], DatanodeInfoWithStorage[127.0.0.1:42095,DS-57938881-b63d-4ce8-abdb-68cff7369150,DISK]] 2024-12-02T09:17:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38963 {}] regionserver.HRegion(8855): Flush requested on d702b350e36b4ab2fde0db91f63c1c5d 2024-12-02T09:17:29,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d702b350e36b4ab2fde0db91f63c1c5d 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-02T09:17:29,751 INFO [FSHLog-0-hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a-prefix:7c6d666a4939,38963,1733131010177 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43645,DS-549e9a62-8573-4a21-80cf-c6eb88542325,DISK], DatanodeInfoWithStorage[127.0.0.1:42095,DS-57938881-b63d-4ce8-abdb-68cff7369150,DISK]] 2024-12-02T09:17:29,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/.tmp/info/fae320025c3d4876b4fbe2756112808f is 1080, key is row0008/info:/1733131036514/Put/seqid=0 2024-12-02T09:17:29,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741840_1016 (size=12509) 2024-12-02T09:17:29,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741840_1016 (size=12509) 2024-12-02T09:17:29,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/.tmp/info/fae320025c3d4876b4fbe2756112808f 2024-12-02T09:17:29,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/.tmp/info/fae320025c3d4876b4fbe2756112808f as hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/fae320025c3d4876b4fbe2756112808f 2024-12-02T09:17:29,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/fae320025c3d4876b4fbe2756112808f, entries=7, sequenceid=21, filesize=12.2 K 2024-12-02T09:17:29,986 INFO [FSHLog-0-hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a-prefix:7c6d666a4939,38963,1733131010177 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43645,DS-549e9a62-8573-4a21-80cf-c6eb88542325,DISK], DatanodeInfoWithStorage[127.0.0.1:42095,DS-57938881-b63d-4ce8-abdb-68cff7369150,DISK]] 2024-12-02T09:17:29,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for d702b350e36b4ab2fde0db91f63c1c5d in 437ms, sequenceid=21, compaction requested=false 2024-12-02T09:17:29,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d702b350e36b4ab2fde0db91f63c1c5d: 2024-12-02T09:17:29,986 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K 2024-12-02T09:17:29,986 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-02T09:17:29,987 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/a5f327fda3de443192e2ca39aaaa205a because midkey is the same as first or last row 2024-12-02T09:17:31,753 INFO [FSHLog-0-hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a-prefix:7c6d666a4939,38963,1733131010177 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43645,DS-549e9a62-8573-4a21-80cf-c6eb88542325,DISK], DatanodeInfoWithStorage[127.0.0.1:42095,DS-57938881-b63d-4ce8-abdb-68cff7369150,DISK]] 2024-12-02T09:17:32,214 INFO [master/7c6d666a4939:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-02T09:17:32,214 INFO [master/7c6d666a4939:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-02T09:17:33,957 INFO [FSHLog-0-hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a-prefix:7c6d666a4939,38963,1733131010177 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43645,DS-549e9a62-8573-4a21-80cf-c6eb88542325,DISK], DatanodeInfoWithStorage[127.0.0.1:42095,DS-57938881-b63d-4ce8-abdb-68cff7369150,DISK]] 2024-12-02T09:17:33,959 WARN [FSHLog-0-hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a-prefix:7c6d666a4939,38963,1733131010177 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43645,DS-549e9a62-8573-4a21-80cf-c6eb88542325,DISK], DatanodeInfoWithStorage[127.0.0.1:42095,DS-57938881-b63d-4ce8-abdb-68cff7369150,DISK]] 2024-12-02T09:17:33,960 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 7c6d666a4939%2C38963%2C1733131010177:(num 1733131042524) roll requested 2024-12-02T09:17:33,960 INFO [regionserver/7c6d666a4939:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C38963%2C1733131010177.1733131053960 2024-12-02T09:17:34,169 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43645,DS-549e9a62-8573-4a21-80cf-c6eb88542325,DISK], DatanodeInfoWithStorage[127.0.0.1:42095,DS-57938881-b63d-4ce8-abdb-68cff7369150,DISK]] 2024-12-02T09:17:34,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:17:34,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:17:34,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:17:34,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:17:34,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:17:34,170 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131042524 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131053960 2024-12-02T09:17:34,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741839_1015 (size=7739) 2024-12-02T09:17:34,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741839_1015 (size=7739) 2024-12-02T09:17:34,176 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38527:38527),(127.0.0.1/127.0.0.1:43595:43595)] 2024-12-02T09:17:34,176 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131042524 is not closed yet, will try archiving it next time 2024-12-02T09:17:34,176 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131022449 to hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/oldWALs/7c6d666a4939%2C38963%2C1733131010177.1733131022449 2024-12-02T09:17:36,163 INFO [FSHLog-0-hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a-prefix:7c6d666a4939,38963,1733131010177 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42095,DS-57938881-b63d-4ce8-abdb-68cff7369150,DISK], DatanodeInfoWithStorage[127.0.0.1:43645,DS-549e9a62-8573-4a21-80cf-c6eb88542325,DISK]] 2024-12-02T09:17:37,815 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region d702b350e36b4ab2fde0db91f63c1c5d, had cached 0 bytes from a total of 25018 2024-12-02T09:17:38,371 INFO [FSHLog-0-hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a-prefix:7c6d666a4939,38963,1733131010177 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42095,DS-57938881-b63d-4ce8-abdb-68cff7369150,DISK], DatanodeInfoWithStorage[127.0.0.1:43645,DS-549e9a62-8573-4a21-80cf-c6eb88542325,DISK]] 2024-12-02T09:17:40,578 INFO [FSHLog-0-hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a-prefix:7c6d666a4939,38963,1733131010177 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42095,DS-57938881-b63d-4ce8-abdb-68cff7369150,DISK], DatanodeInfoWithStorage[127.0.0.1:43645,DS-549e9a62-8573-4a21-80cf-c6eb88542325,DISK]] 2024-12-02T09:17:42,783 INFO [FSHLog-0-hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a-prefix:7c6d666a4939,38963,1733131010177 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42095,DS-57938881-b63d-4ce8-abdb-68cff7369150,DISK], DatanodeInfoWithStorage[127.0.0.1:43645,DS-549e9a62-8573-4a21-80cf-c6eb88542325,DISK]] 2024-12-02T09:17:44,785 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-12-02T09:17:44,786 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C38963%2C1733131010177.1733131064785 2024-12-02T09:17:48,585 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T09:17:49,802 INFO [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5013 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42095,DS-57938881-b63d-4ce8-abdb-68cff7369150,DISK], DatanodeInfoWithStorage[127.0.0.1:43645,DS-549e9a62-8573-4a21-80cf-c6eb88542325,DISK]] 2024-12-02T09:17:49,804 WARN [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5013 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:42095,DS-57938881-b63d-4ce8-abdb-68cff7369150,DISK], DatanodeInfoWithStorage[127.0.0.1:43645,DS-549e9a62-8573-4a21-80cf-c6eb88542325,DISK]] 2024-12-02T09:17:49,804 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 7c6d666a4939%2C38963%2C1733131010177:(num 1733131064785) roll requested 2024-12-02T09:17:49,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:17:49,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:17:49,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:17:49,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:17:49,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:17:49,806 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131053960 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131064785 2024-12-02T09:17:49,807 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43595:43595),(127.0.0.1/127.0.0.1:38527:38527)] 2024-12-02T09:17:49,807 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131053960 is not closed yet, will try archiving it next time 2024-12-02T09:17:49,808 INFO [regionserver/7c6d666a4939:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C38963%2C1733131010177.1733131069807 2024-12-02T09:17:49,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741841_1017 (size=4753) 2024-12-02T09:17:49,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741841_1017 (size=4753) 2024-12-02T09:17:54,812 INFO [FSHLog-0-hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a-prefix:7c6d666a4939,38963,1733131010177 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5002 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43645,DS-549e9a62-8573-4a21-80cf-c6eb88542325,DISK], DatanodeInfoWithStorage[127.0.0.1:42095,DS-57938881-b63d-4ce8-abdb-68cff7369150,DISK]] 2024-12-02T09:17:54,812 WARN [FSHLog-0-hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a-prefix:7c6d666a4939,38963,1733131010177 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5002 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43645,DS-549e9a62-8573-4a21-80cf-c6eb88542325,DISK], DatanodeInfoWithStorage[127.0.0.1:42095,DS-57938881-b63d-4ce8-abdb-68cff7369150,DISK]] 2024-12-02T09:17:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38963 {}] regionserver.HRegion(8855): Flush requested on d702b350e36b4ab2fde0db91f63c1c5d 2024-12-02T09:17:54,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d702b350e36b4ab2fde0db91f63c1c5d 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-02T09:17:54,852 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5039 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43645,DS-549e9a62-8573-4a21-80cf-c6eb88542325,DISK], DatanodeInfoWithStorage[127.0.0.1:42095,DS-57938881-b63d-4ce8-abdb-68cff7369150,DISK]] 2024-12-02T09:17:54,852 WARN [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5039 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43645,DS-549e9a62-8573-4a21-80cf-c6eb88542325,DISK], DatanodeInfoWithStorage[127.0.0.1:42095,DS-57938881-b63d-4ce8-abdb-68cff7369150,DISK]] 2024-12-02T09:17:56,814 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-12-02T09:17:59,817 INFO [FSHLog-0-hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a-prefix:7c6d666a4939,38963,1733131010177 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43645,DS-549e9a62-8573-4a21-80cf-c6eb88542325,DISK], DatanodeInfoWithStorage[127.0.0.1:42095,DS-57938881-b63d-4ce8-abdb-68cff7369150,DISK]] 2024-12-02T09:17:59,817 WARN [FSHLog-0-hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a-prefix:7c6d666a4939,38963,1733131010177 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43645,DS-549e9a62-8573-4a21-80cf-c6eb88542325,DISK], DatanodeInfoWithStorage[127.0.0.1:42095,DS-57938881-b63d-4ce8-abdb-68cff7369150,DISK]] 2024-12-02T09:17:59,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:17:59,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:17:59,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:17:59,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:17:59,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:17:59,818 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131064785 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131069807 2024-12-02T09:17:59,819 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43595:43595),(127.0.0.1/127.0.0.1:38527:38527)] 2024-12-02T09:17:59,819 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131064785 is not closed yet, will try archiving it next time 2024-12-02T09:17:59,820 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 7c6d666a4939%2C38963%2C1733131010177:(num 1733131069807) roll requested 2024-12-02T09:17:59,820 INFO [regionserver/7c6d666a4939:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C38963%2C1733131010177.1733131079820 2024-12-02T09:17:59,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741842_1018 (size=1569) 2024-12-02T09:17:59,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741842_1018 (size=1569) 2024-12-02T09:17:59,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/.tmp/info/25750dc0275a4d198089ba8fb9c5d01a is 1080, key is row0015/info:/1733131051551/Put/seqid=0 2024-12-02T09:17:59,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741844_1020 (size=12509) 2024-12-02T09:17:59,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741844_1020 (size=12509) 2024-12-02T09:17:59,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/.tmp/info/25750dc0275a4d198089ba8fb9c5d01a 2024-12-02T09:17:59,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/.tmp/info/25750dc0275a4d198089ba8fb9c5d01a as hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/25750dc0275a4d198089ba8fb9c5d01a 2024-12-02T09:17:59,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/25750dc0275a4d198089ba8fb9c5d01a, entries=7, sequenceid=31, filesize=12.2 K 2024-12-02T09:18:04,836 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5013 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43645,DS-549e9a62-8573-4a21-80cf-c6eb88542325,DISK], DatanodeInfoWithStorage[127.0.0.1:42095,DS-57938881-b63d-4ce8-abdb-68cff7369150,DISK]] 2024-12-02T09:18:04,836 WARN [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5013 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43645,DS-549e9a62-8573-4a21-80cf-c6eb88542325,DISK], DatanodeInfoWithStorage[127.0.0.1:42095,DS-57938881-b63d-4ce8-abdb-68cff7369150,DISK]] 2024-12-02T09:18:04,855 INFO [FSHLog-0-hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a-prefix:7c6d666a4939,38963,1733131010177 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43645,DS-549e9a62-8573-4a21-80cf-c6eb88542325,DISK], DatanodeInfoWithStorage[127.0.0.1:42095,DS-57938881-b63d-4ce8-abdb-68cff7369150,DISK]] 2024-12-02T09:18:04,855 WARN [FSHLog-0-hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a-prefix:7c6d666a4939,38963,1733131010177 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43645,DS-549e9a62-8573-4a21-80cf-c6eb88542325,DISK], DatanodeInfoWithStorage[127.0.0.1:42095,DS-57938881-b63d-4ce8-abdb-68cff7369150,DISK]] 2024-12-02T09:18:04,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for d702b350e36b4ab2fde0db91f63c1c5d in 10042ms, sequenceid=31, compaction requested=true 2024-12-02T09:18:04,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:04,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d702b350e36b4ab2fde0db91f63c1c5d: 2024-12-02T09:18:04,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:04,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:04,856 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K 2024-12-02T09:18:04,856 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-02T09:18:04,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:04,856 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/a5f327fda3de443192e2ca39aaaa205a because midkey is the same as first or last row 2024-12-02T09:18:04,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:04,856 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131069807 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131079820 2024-12-02T09:18:04,857 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38527:38527),(127.0.0.1/127.0.0.1:43595:43595)] 2024-12-02T09:18:04,857 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131069807 is not closed yet, will try archiving it next time 2024-12-02T09:18:04,857 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131042524 to hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/oldWALs/7c6d666a4939%2C38963%2C1733131010177.1733131042524 2024-12-02T09:18:04,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d702b350e36b4ab2fde0db91f63c1c5d:info, priority=-2147483648, current under compaction store size is 1 2024-12-02T09:18:04,858 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 7c6d666a4939%2C38963%2C1733131010177:(num 1733131084858) roll requested 2024-12-02T09:18:04,858 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C38963%2C1733131010177.1733131084858 2024-12-02T09:18:04,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741843_1019 (size=438) 2024-12-02T09:18:04,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741843_1019 (size=438) 2024-12-02T09:18:04,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:18:04,862 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131053960 to hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/oldWALs/7c6d666a4939%2C38963%2C1733131010177.1733131053960 2024-12-02T09:18:04,862 DEBUG [RS:0;7c6d666a4939:38963-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T09:18:04,865 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131064785 to hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/oldWALs/7c6d666a4939%2C38963%2C1733131010177.1733131064785 2024-12-02T09:18:04,867 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131069807 to hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/oldWALs/7c6d666a4939%2C38963%2C1733131010177.1733131069807 2024-12-02T09:18:04,867 DEBUG [RS:0;7c6d666a4939:38963-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T09:18:04,869 DEBUG [RS:0;7c6d666a4939:38963-shortCompactions-0 {}] regionserver.HStore(1541): d702b350e36b4ab2fde0db91f63c1c5d/info is initiating minor compaction (all files) 2024-12-02T09:18:04,869 INFO [RS:0;7c6d666a4939:38963-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d702b350e36b4ab2fde0db91f63c1c5d/info in TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d. 2024-12-02T09:18:04,869 INFO [RS:0;7c6d666a4939:38963-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/a5f327fda3de443192e2ca39aaaa205a, hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/fae320025c3d4876b4fbe2756112808f, hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/25750dc0275a4d198089ba8fb9c5d01a] into tmpdir=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/.tmp, totalSize=36.6 K 2024-12-02T09:18:04,871 DEBUG [RS:0;7c6d666a4939:38963-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5f327fda3de443192e2ca39aaaa205a, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733131022473 2024-12-02T09:18:04,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:04,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:04,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:04,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:04,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:04,871 DEBUG [RS:0;7c6d666a4939:38963-shortCompactions-0 {}] compactions.Compactor(225): Compacting fae320025c3d4876b4fbe2756112808f, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1733131036514 2024-12-02T09:18:04,871 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131079820 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131084858 2024-12-02T09:18:04,872 DEBUG [RS:0;7c6d666a4939:38963-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25750dc0275a4d198089ba8fb9c5d01a, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1733131051551 2024-12-02T09:18:04,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741845_1021 (size=93) 2024-12-02T09:18:04,875 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131079820 to hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/oldWALs/7c6d666a4939%2C38963%2C1733131010177.1733131079820 2024-12-02T09:18:04,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741845_1021 (size=93) 2024-12-02T09:18:04,884 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38527:38527),(127.0.0.1/127.0.0.1:43595:43595)] 2024-12-02T09:18:04,885 INFO [regionserver/7c6d666a4939:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C38963%2C1733131010177.1733131084884 2024-12-02T09:18:04,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:04,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:04,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:04,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:04,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:04,912 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131084858 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/WALs/7c6d666a4939,38963,1733131010177/7c6d666a4939%2C38963%2C1733131010177.1733131084884 2024-12-02T09:18:04,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741846_1022 (size=1258) 2024-12-02T09:18:04,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741846_1022 (size=1258) 2024-12-02T09:18:04,922 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43595:43595),(127.0.0.1/127.0.0.1:38527:38527)] 2024-12-02T09:18:04,927 INFO [RS:0;7c6d666a4939:38963-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d702b350e36b4ab2fde0db91f63c1c5d#info#compaction#3 average throughput is 7.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T09:18:04,929 DEBUG [RS:0;7c6d666a4939:38963-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/.tmp/info/02c7321f46dc42d09f56f0beb94fac57 is 1080, key is row0001/info:/1733131022473/Put/seqid=0 2024-12-02T09:18:04,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741848_1024 (size=27710) 2024-12-02T09:18:04,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741848_1024 (size=27710) 2024-12-02T09:18:04,948 DEBUG [RS:0;7c6d666a4939:38963-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/.tmp/info/02c7321f46dc42d09f56f0beb94fac57 as hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/02c7321f46dc42d09f56f0beb94fac57 2024-12-02T09:18:04,966 INFO [RS:0;7c6d666a4939:38963-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d702b350e36b4ab2fde0db91f63c1c5d/info of d702b350e36b4ab2fde0db91f63c1c5d into 02c7321f46dc42d09f56f0beb94fac57(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T09:18:04,966 DEBUG [RS:0;7c6d666a4939:38963-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d702b350e36b4ab2fde0db91f63c1c5d: 2024-12-02T09:18:04,969 INFO [RS:0;7c6d666a4939:38963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d., storeName=d702b350e36b4ab2fde0db91f63c1c5d/info, priority=13, startTime=1733131084857; duration=0sec 2024-12-02T09:18:04,969 DEBUG [RS:0;7c6d666a4939:38963-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K 2024-12-02T09:18:04,969 DEBUG [RS:0;7c6d666a4939:38963-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-02T09:18:04,969 DEBUG [RS:0;7c6d666a4939:38963-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/02c7321f46dc42d09f56f0beb94fac57 because midkey is the same as first or last row 2024-12-02T09:18:04,970 DEBUG [RS:0;7c6d666a4939:38963-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K 2024-12-02T09:18:04,970 DEBUG [RS:0;7c6d666a4939:38963-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-02T09:18:04,970 DEBUG [RS:0;7c6d666a4939:38963-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/02c7321f46dc42d09f56f0beb94fac57 because midkey is the same as first or last row 2024-12-02T09:18:04,971 DEBUG [RS:0;7c6d666a4939:38963-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K 2024-12-02T09:18:04,971 DEBUG [RS:0;7c6d666a4939:38963-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-02T09:18:04,971 DEBUG [RS:0;7c6d666a4939:38963-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/02c7321f46dc42d09f56f0beb94fac57 because midkey is the same as first or last row 2024-12-02T09:18:04,971 DEBUG [RS:0;7c6d666a4939:38963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:18:04,971 DEBUG [RS:0;7c6d666a4939:38963-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d702b350e36b4ab2fde0db91f63c1c5d:info 2024-12-02T09:18:16,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38963 {}] regionserver.HRegion(8855): Flush requested on d702b350e36b4ab2fde0db91f63c1c5d 2024-12-02T09:18:16,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d702b350e36b4ab2fde0db91f63c1c5d 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-02T09:18:16,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/.tmp/info/202377dd3e074dbbb461ba268a61e9e1 is 1080, key is row0022/info:/1733131084886/Put/seqid=0 2024-12-02T09:18:16,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741849_1025 (size=12509) 2024-12-02T09:18:16,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741849_1025 (size=12509) 2024-12-02T09:18:16,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/.tmp/info/202377dd3e074dbbb461ba268a61e9e1 2024-12-02T09:18:16,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/.tmp/info/202377dd3e074dbbb461ba268a61e9e1 as hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/202377dd3e074dbbb461ba268a61e9e1 2024-12-02T09:18:16,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/202377dd3e074dbbb461ba268a61e9e1, entries=7, sequenceid=42, filesize=12.2 K 2024-12-02T09:18:16,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for d702b350e36b4ab2fde0db91f63c1c5d in 52ms, sequenceid=42, compaction requested=false 2024-12-02T09:18:16,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d702b350e36b4ab2fde0db91f63c1c5d: 2024-12-02T09:18:16,961 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K 2024-12-02T09:18:16,961 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-02T09:18:16,961 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/02c7321f46dc42d09f56f0beb94fac57 because midkey is the same as first or last row 2024-12-02T09:18:18,585 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T09:18:22,816 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region d702b350e36b4ab2fde0db91f63c1c5d, had cached 0 bytes from a total of 40219 2024-12-02T09:18:24,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-02T09:18:24,922 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T09:18:24,922 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T09:18:24,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:18:24,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:18:24,927 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T09:18:24,927 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-02T09:18:24,927 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=372944084, stopped=false 2024-12-02T09:18:24,927 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7c6d666a4939,43095,1733131009439 2024-12-02T09:18:24,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38963-0x1009a4659ad0001, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T09:18:24,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T09:18:24,989 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T09:18:24,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38963-0x1009a4659ad0001, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:24,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:24,989 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T09:18:24,989 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T09:18:24,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:18:24,990 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:18:24,990 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7c6d666a4939,38963,1733131010177' ***** 2024-12-02T09:18:24,990 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-02T09:18:24,991 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38963-0x1009a4659ad0001, quorum=127.0.0.1:53155, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:18:24,991 INFO [RS:0;7c6d666a4939:38963 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T09:18:24,992 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T09:18:24,992 INFO [RS:0;7c6d666a4939:38963 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-02T09:18:24,992 INFO [RS:0;7c6d666a4939:38963 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-02T09:18:24,992 INFO [RS:0;7c6d666a4939:38963 {}] regionserver.HRegionServer(3091): Received CLOSE for d702b350e36b4ab2fde0db91f63c1c5d 2024-12-02T09:18:24,993 INFO [RS:0;7c6d666a4939:38963 {}] regionserver.HRegionServer(959): stopping server 7c6d666a4939,38963,1733131010177 2024-12-02T09:18:24,993 INFO [RS:0;7c6d666a4939:38963 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T09:18:24,993 INFO [RS:0;7c6d666a4939:38963 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7c6d666a4939:38963. 2024-12-02T09:18:24,993 DEBUG [RS:0;7c6d666a4939:38963 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T09:18:24,993 DEBUG [RS:0;7c6d666a4939:38963 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:18:24,993 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d702b350e36b4ab2fde0db91f63c1c5d, disabling compactions & flushes 2024-12-02T09:18:24,993 INFO [RS:0;7c6d666a4939:38963 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T09:18:24,994 INFO [RS:0;7c6d666a4939:38963 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T09:18:24,994 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d. 2024-12-02T09:18:24,994 INFO [RS:0;7c6d666a4939:38963 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T09:18:24,994 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d. 2024-12-02T09:18:24,994 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d. after waiting 0 ms 2024-12-02T09:18:24,994 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d. 2024-12-02T09:18:24,994 INFO [RS:0;7c6d666a4939:38963 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-02T09:18:24,994 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d702b350e36b4ab2fde0db91f63c1c5d 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB 2024-12-02T09:18:24,994 INFO [RS:0;7c6d666a4939:38963 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-02T09:18:24,994 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T09:18:24,994 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T09:18:24,995 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T09:18:24,995 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T09:18:24,995 DEBUG [RS:0;7c6d666a4939:38963 {}] regionserver.HRegionServer(1325): Online Regions={d702b350e36b4ab2fde0db91f63c1c5d=TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d., 1588230740=hbase:meta,,1.1588230740} 2024-12-02T09:18:24,995 DEBUG [RS:0;7c6d666a4939:38963 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, d702b350e36b4ab2fde0db91f63c1c5d 2024-12-02T09:18:24,995 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T09:18:24,995 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB 2024-12-02T09:18:25,006 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/.tmp/info/db4bc53b031840ce9a901734dc58643a is 1080, key is row0029/info:/1733131098910/Put/seqid=0 2024-12-02T09:18:25,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741850_1026 (size=8193) 2024-12-02T09:18:25,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741850_1026 (size=8193) 2024-12-02T09:18:25,016 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/.tmp/info/db4bc53b031840ce9a901734dc58643a 2024-12-02T09:18:25,025 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/hbase/meta/1588230740/.tmp/info/04e810d09033436091f179663ab28926 is 195, key is TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d./info:regioninfo/1733131012845/Put/seqid=0 2024-12-02T09:18:25,038 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/.tmp/info/db4bc53b031840ce9a901734dc58643a as hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/db4bc53b031840ce9a901734dc58643a 2024-12-02T09:18:25,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741851_1027 (size=7016) 2024-12-02T09:18:25,053 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/hbase/meta/1588230740/.tmp/info/04e810d09033436091f179663ab28926 2024-12-02T09:18:25,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741851_1027 (size=7016) 2024-12-02T09:18:25,060 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/db4bc53b031840ce9a901734dc58643a, entries=3, sequenceid=48, filesize=8.0 K 2024-12-02T09:18:25,062 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for d702b350e36b4ab2fde0db91f63c1c5d in 68ms, sequenceid=48, compaction requested=true 2024-12-02T09:18:25,063 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/a5f327fda3de443192e2ca39aaaa205a, hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/fae320025c3d4876b4fbe2756112808f, hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/25750dc0275a4d198089ba8fb9c5d01a] to archive 2024-12-02T09:18:25,067 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T09:18:25,072 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/a5f327fda3de443192e2ca39aaaa205a to hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/archive/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/a5f327fda3de443192e2ca39aaaa205a 2024-12-02T09:18:25,075 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/fae320025c3d4876b4fbe2756112808f to hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/archive/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/fae320025c3d4876b4fbe2756112808f 2024-12-02T09:18:25,091 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/hbase/meta/1588230740/.tmp/ns/29f57601a29947e3b3f5285c63b866ea is 43, key is default/ns:d/1733131012124/Put/seqid=0 2024-12-02T09:18:25,091 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/25750dc0275a4d198089ba8fb9c5d01a to hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/archive/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/info/25750dc0275a4d198089ba8fb9c5d01a 2024-12-02T09:18:25,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741852_1028 (size=5153) 2024-12-02T09:18:25,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741852_1028 (size=5153) 2024-12-02T09:18:25,111 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=7c6d666a4939:43095 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-12-02T09:18:25,116 WARN [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [a5f327fda3de443192e2ca39aaaa205a=12509, fae320025c3d4876b4fbe2756112808f=12509, 25750dc0275a4d198089ba8fb9c5d01a=12509] 2024-12-02T09:18:25,116 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/hbase/meta/1588230740/.tmp/ns/29f57601a29947e3b3f5285c63b866ea 2024-12-02T09:18:25,127 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/default/TestLogRolling-testSlowSyncLogRolling/d702b350e36b4ab2fde0db91f63c1c5d/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1 2024-12-02T09:18:25,136 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d. 2024-12-02T09:18:25,136 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d702b350e36b4ab2fde0db91f63c1c5d: Waiting for close lock at 1733131104993Running coprocessor pre-close hooks at 1733131104993Disabling compacts and flushes for region at 1733131104993Disabling writes for close at 1733131104994 (+1 ms)Obtaining lock to block concurrent updates at 1733131104994Preparing flush snapshotting stores in d702b350e36b4ab2fde0db91f63c1c5d at 1733131104994Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1733131104995 (+1 ms)Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d. at 1733131104999 (+4 ms)Flushing d702b350e36b4ab2fde0db91f63c1c5d/info: creating writer at 1733131104999Flushing d702b350e36b4ab2fde0db91f63c1c5d/info: appending metadata at 1733131105006 (+7 ms)Flushing d702b350e36b4ab2fde0db91f63c1c5d/info: closing flushed file at 1733131105006Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6252c15b: reopening flushed file at 1733131105037 (+31 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for d702b350e36b4ab2fde0db91f63c1c5d in 68ms, sequenceid=48, compaction requested=true at 1733131105062 (+25 ms)Writing region close event to WAL at 1733131105117 (+55 ms)Running coprocessor post-close hooks at 1733131105133 (+16 ms)Closed at 1733131105136 (+3 ms) 2024-12-02T09:18:25,137 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1733131012358.d702b350e36b4ab2fde0db91f63c1c5d. 2024-12-02T09:18:25,184 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/hbase/meta/1588230740/.tmp/table/aa9a15aa7a1c4a5682e153b3f5bd45d2 is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1733131012864/Put/seqid=0 2024-12-02T09:18:25,195 DEBUG [RS:0;7c6d666a4939:38963 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T09:18:25,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741853_1029 (size=5396) 2024-12-02T09:18:25,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741853_1029 (size=5396) 2024-12-02T09:18:25,208 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/hbase/meta/1588230740/.tmp/table/aa9a15aa7a1c4a5682e153b3f5bd45d2 2024-12-02T09:18:25,218 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/hbase/meta/1588230740/.tmp/info/04e810d09033436091f179663ab28926 as hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/hbase/meta/1588230740/info/04e810d09033436091f179663ab28926 2024-12-02T09:18:25,236 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/hbase/meta/1588230740/info/04e810d09033436091f179663ab28926, entries=10, sequenceid=11, filesize=6.9 K 2024-12-02T09:18:25,239 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/hbase/meta/1588230740/.tmp/ns/29f57601a29947e3b3f5285c63b866ea as hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/hbase/meta/1588230740/ns/29f57601a29947e3b3f5285c63b866ea 2024-12-02T09:18:25,248 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/hbase/meta/1588230740/ns/29f57601a29947e3b3f5285c63b866ea, entries=2, sequenceid=11, filesize=5.0 K 2024-12-02T09:18:25,249 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/hbase/meta/1588230740/.tmp/table/aa9a15aa7a1c4a5682e153b3f5bd45d2 as hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/hbase/meta/1588230740/table/aa9a15aa7a1c4a5682e153b3f5bd45d2 2024-12-02T09:18:25,259 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/hbase/meta/1588230740/table/aa9a15aa7a1c4a5682e153b3f5bd45d2, entries=2, sequenceid=11, filesize=5.3 K 2024-12-02T09:18:25,261 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 265ms, sequenceid=11, compaction requested=false 2024-12-02T09:18:25,285 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-02T09:18:25,287 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T09:18:25,287 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T09:18:25,287 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733131104994Running coprocessor pre-close hooks at 1733131104994Disabling compacts and flushes for region at 1733131104994Disabling writes for close at 1733131104995 (+1 ms)Obtaining lock to block concurrent updates at 1733131104995Preparing flush snapshotting stores in 1588230740 at 1733131104995Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1733131104996 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733131104997 (+1 ms)Flushing 1588230740/info: creating writer at 1733131104997Flushing 1588230740/info: appending metadata at 1733131105025 (+28 ms)Flushing 1588230740/info: closing flushed file at 1733131105025Flushing 1588230740/ns: creating writer at 1733131105068 (+43 ms)Flushing 1588230740/ns: appending metadata at 1733131105090 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1733131105090Flushing 1588230740/table: creating writer at 1733131105141 (+51 ms)Flushing 1588230740/table: appending metadata at 1733131105183 (+42 ms)Flushing 1588230740/table: closing flushed file at 1733131105183Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@201908f7: reopening flushed file at 1733131105217 (+34 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f31050c: reopening flushed file at 1733131105236 (+19 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e518e31: reopening flushed file at 1733131105248 (+12 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 265ms, sequenceid=11, compaction requested=false at 1733131105261 (+13 ms)Writing region close event to WAL at 1733131105272 (+11 ms)Running coprocessor post-close hooks at 1733131105287 (+15 ms)Closed at 1733131105287 2024-12-02T09:18:25,288 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-02T09:18:25,396 INFO [RS:0;7c6d666a4939:38963 {}] regionserver.HRegionServer(976): stopping server 7c6d666a4939,38963,1733131010177; all regions closed. 2024-12-02T09:18:25,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:25,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:25,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:25,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:25,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:25,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741834_1010 (size=3066) 2024-12-02T09:18:25,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741834_1010 (size=3066) 2024-12-02T09:18:25,403 INFO [regionserver/7c6d666a4939:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T09:18:25,405 DEBUG [RS:0;7c6d666a4939:38963 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/oldWALs 2024-12-02T09:18:25,405 INFO [RS:0;7c6d666a4939:38963 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7c6d666a4939%2C38963%2C1733131010177.meta:.meta(num 1733131011949) 2024-12-02T09:18:25,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:25,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:25,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:25,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:25,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:25,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741847_1023 (size=12695) 2024-12-02T09:18:25,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741847_1023 (size=12695) 2024-12-02T09:18:25,413 DEBUG [RS:0;7c6d666a4939:38963 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/oldWALs 2024-12-02T09:18:25,413 INFO [RS:0;7c6d666a4939:38963 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7c6d666a4939%2C38963%2C1733131010177:(num 1733131084884) 2024-12-02T09:18:25,414 DEBUG [RS:0;7c6d666a4939:38963 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:18:25,414 INFO [RS:0;7c6d666a4939:38963 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T09:18:25,414 INFO [RS:0;7c6d666a4939:38963 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T09:18:25,414 INFO [RS:0;7c6d666a4939:38963 {}] hbase.ChoreService(370): Chore service for: regionserver/7c6d666a4939:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-02T09:18:25,414 INFO [RS:0;7c6d666a4939:38963 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T09:18:25,414 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T09:18:25,415 INFO [RS:0;7c6d666a4939:38963 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38963 2024-12-02T09:18:25,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T09:18:25,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38963-0x1009a4659ad0001, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7c6d666a4939,38963,1733131010177 2024-12-02T09:18:25,759 INFO [RS:0;7c6d666a4939:38963 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T09:18:25,910 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7c6d666a4939,38963,1733131010177] 2024-12-02T09:18:25,986 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7c6d666a4939,38963,1733131010177 already deleted, retry=false 2024-12-02T09:18:25,986 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7c6d666a4939,38963,1733131010177 expired; onlineServers=0 2024-12-02T09:18:25,986 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7c6d666a4939,43095,1733131009439' ***** 2024-12-02T09:18:25,986 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-02T09:18:25,987 INFO [M:0;7c6d666a4939:43095 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T09:18:25,987 INFO [M:0;7c6d666a4939:43095 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T09:18:25,987 DEBUG [M:0;7c6d666a4939:43095 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-02T09:18:25,987 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-02T09:18:25,987 DEBUG [M:0;7c6d666a4939:43095 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-02T09:18:25,987 DEBUG [master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.large.0-1733131011190 {}] cleaner.HFileCleaner(306): Exit Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.large.0-1733131011190,5,FailOnTimeoutGroup] 2024-12-02T09:18:25,987 DEBUG [master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.small.0-1733131011192 {}] cleaner.HFileCleaner(306): Exit Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.small.0-1733131011192,5,FailOnTimeoutGroup] 2024-12-02T09:18:25,987 INFO [M:0;7c6d666a4939:43095 {}] hbase.ChoreService(370): Chore service for: master/7c6d666a4939:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-02T09:18:25,987 INFO [M:0;7c6d666a4939:43095 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T09:18:25,987 DEBUG [M:0;7c6d666a4939:43095 {}] master.HMaster(1795): Stopping service threads 2024-12-02T09:18:25,987 INFO [M:0;7c6d666a4939:43095 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-02T09:18:25,988 INFO [M:0;7c6d666a4939:43095 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T09:18:25,988 INFO [M:0;7c6d666a4939:43095 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T09:18:25,988 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-02T09:18:26,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38963-0x1009a4659ad0001, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:18:26,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38963-0x1009a4659ad0001, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:18:26,011 INFO [RS:0;7c6d666a4939:38963 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T09:18:26,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-02T09:18:26,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:26,011 INFO [RS:0;7c6d666a4939:38963 {}] regionserver.HRegionServer(1031): Exiting; stopping=7c6d666a4939,38963,1733131010177; zookeeper connection closed. 2024-12-02T09:18:26,011 DEBUG [M:0;7c6d666a4939:43095 {}] zookeeper.ZKUtil(347): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-02T09:18:26,011 WARN [M:0;7c6d666a4939:43095 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-02T09:18:26,011 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5407464d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5407464d 2024-12-02T09:18:26,012 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-02T09:18:26,012 INFO [M:0;7c6d666a4939:43095 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/.lastflushedseqids 2024-12-02T09:18:26,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741854_1030 (size=130) 2024-12-02T09:18:26,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741854_1030 (size=130) 2024-12-02T09:18:26,027 INFO [M:0;7c6d666a4939:43095 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T09:18:26,027 INFO [M:0;7c6d666a4939:43095 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-02T09:18:26,027 DEBUG [M:0;7c6d666a4939:43095 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T09:18:26,027 INFO [M:0;7c6d666a4939:43095 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:18:26,027 DEBUG [M:0;7c6d666a4939:43095 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:18:26,027 DEBUG [M:0;7c6d666a4939:43095 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T09:18:26,027 DEBUG [M:0;7c6d666a4939:43095 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:18:26,027 INFO [M:0;7c6d666a4939:43095 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.02 KB heapSize=29.20 KB 2024-12-02T09:18:26,049 DEBUG [M:0;7c6d666a4939:43095 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a2cdd86e13eb4a429b829009ea22d619 is 82, key is hbase:meta,,1/info:regioninfo/1733131012024/Put/seqid=0 2024-12-02T09:18:26,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741855_1031 (size=5672) 2024-12-02T09:18:26,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741855_1031 (size=5672) 2024-12-02T09:18:26,056 INFO [M:0;7c6d666a4939:43095 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a2cdd86e13eb4a429b829009ea22d619 2024-12-02T09:18:26,078 DEBUG [M:0;7c6d666a4939:43095 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7b9b48b9686a44629c4e872b7dd2ca0e is 766, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733131012872/Put/seqid=0 2024-12-02T09:18:26,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741856_1032 (size=6247) 2024-12-02T09:18:26,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741856_1032 (size=6247) 2024-12-02T09:18:26,084 INFO [M:0;7c6d666a4939:43095 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7b9b48b9686a44629c4e872b7dd2ca0e 2024-12-02T09:18:26,093 INFO [M:0;7c6d666a4939:43095 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7b9b48b9686a44629c4e872b7dd2ca0e 2024-12-02T09:18:26,114 DEBUG [M:0;7c6d666a4939:43095 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dc7ee621f72e4fb1a098c77075377603 is 69, key is 7c6d666a4939,38963,1733131010177/rs:state/1733131011317/Put/seqid=0 2024-12-02T09:18:26,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741857_1033 (size=5156) 2024-12-02T09:18:26,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741857_1033 (size=5156) 2024-12-02T09:18:26,121 INFO [M:0;7c6d666a4939:43095 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dc7ee621f72e4fb1a098c77075377603 2024-12-02T09:18:26,149 DEBUG [M:0;7c6d666a4939:43095 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/57d8f4c97ad1468ab8f5f5e065844bf3 is 52, key is load_balancer_on/state:d/1733131012340/Put/seqid=0 2024-12-02T09:18:26,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741858_1034 (size=5056) 2024-12-02T09:18:26,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741858_1034 (size=5056) 2024-12-02T09:18:26,156 INFO [M:0;7c6d666a4939:43095 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/57d8f4c97ad1468ab8f5f5e065844bf3 2024-12-02T09:18:26,164 DEBUG [M:0;7c6d666a4939:43095 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a2cdd86e13eb4a429b829009ea22d619 as hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a2cdd86e13eb4a429b829009ea22d619 2024-12-02T09:18:26,171 INFO [M:0;7c6d666a4939:43095 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a2cdd86e13eb4a429b829009ea22d619, entries=8, sequenceid=59, filesize=5.5 K 2024-12-02T09:18:26,172 DEBUG [M:0;7c6d666a4939:43095 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7b9b48b9686a44629c4e872b7dd2ca0e as hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7b9b48b9686a44629c4e872b7dd2ca0e 2024-12-02T09:18:26,182 INFO [M:0;7c6d666a4939:43095 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7b9b48b9686a44629c4e872b7dd2ca0e 2024-12-02T09:18:26,182 INFO [M:0;7c6d666a4939:43095 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7b9b48b9686a44629c4e872b7dd2ca0e, entries=6, sequenceid=59, filesize=6.1 K 2024-12-02T09:18:26,184 DEBUG [M:0;7c6d666a4939:43095 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dc7ee621f72e4fb1a098c77075377603 as hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/dc7ee621f72e4fb1a098c77075377603 2024-12-02T09:18:26,191 INFO [M:0;7c6d666a4939:43095 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/dc7ee621f72e4fb1a098c77075377603, entries=1, sequenceid=59, filesize=5.0 K 2024-12-02T09:18:26,193 DEBUG [M:0;7c6d666a4939:43095 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/57d8f4c97ad1468ab8f5f5e065844bf3 as hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/57d8f4c97ad1468ab8f5f5e065844bf3 2024-12-02T09:18:26,200 INFO [M:0;7c6d666a4939:43095 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/57d8f4c97ad1468ab8f5f5e065844bf3, entries=1, sequenceid=59, filesize=4.9 K 2024-12-02T09:18:26,202 INFO [M:0;7c6d666a4939:43095 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 174ms, sequenceid=59, compaction requested=false 2024-12-02T09:18:26,203 INFO [M:0;7c6d666a4939:43095 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:18:26,203 DEBUG [M:0;7c6d666a4939:43095 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733131106027Disabling compacts and flushes for region at 1733131106027Disabling writes for close at 1733131106027Obtaining lock to block concurrent updates at 1733131106027Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733131106027Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23576, getHeapSize=29840, getOffHeapSize=0, getCellsCount=70 at 1733131106028 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733131106029 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733131106029Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733131106049 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733131106049Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733131106063 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733131106077 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733131106077Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733131106093 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733131106114 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733131106114Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733131106128 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733131106148 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733131106148Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e68339b: reopening flushed file at 1733131106163 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75af632: reopening flushed file at 1733131106171 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10455f7a: reopening flushed file at 1733131106182 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ea3e062: reopening flushed file at 1733131106192 (+10 ms)Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 174ms, sequenceid=59, compaction requested=false at 1733131106202 (+10 ms)Writing region close event to WAL at 1733131106203 (+1 ms)Closed at 1733131106203 2024-12-02T09:18:26,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:26,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:26,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:26,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:26,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:26,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42095 is added to blk_1073741830_1006 (size=27973) 2024-12-02T09:18:26,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43645 is added to blk_1073741830_1006 (size=27973) 2024-12-02T09:18:26,207 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T09:18:26,207 INFO [M:0;7c6d666a4939:43095 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-02T09:18:26,208 INFO [M:0;7c6d666a4939:43095 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43095 2024-12-02T09:18:26,208 INFO [M:0;7c6d666a4939:43095 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T09:18:26,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:18:26,317 INFO [M:0;7c6d666a4939:43095 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T09:18:26,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43095-0x1009a4659ad0000, quorum=127.0.0.1:53155, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:18:26,322 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bf97579{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:18:26,324 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:18:26,324 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:18:26,325 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:18:26,325 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/hadoop.log.dir/,STOPPED} 2024-12-02T09:18:26,327 WARN [BP-986440887-172.17.0.3-1733131005458 heartbeating to localhost/127.0.0.1:37857 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T09:18:26,327 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T09:18:26,327 WARN [BP-986440887-172.17.0.3-1733131005458 heartbeating to localhost/127.0.0.1:37857 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-986440887-172.17.0.3-1733131005458 (Datanode Uuid bf87d23c-05b8-41cf-922c-097f536fca77) service to localhost/127.0.0.1:37857 2024-12-02T09:18:26,327 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T09:18:26,329 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/cluster_900e3e1e-6d1a-4bb5-1570-6ffe3d06df03/data/data3/current/BP-986440887-172.17.0.3-1733131005458 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:18:26,329 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/cluster_900e3e1e-6d1a-4bb5-1570-6ffe3d06df03/data/data4/current/BP-986440887-172.17.0.3-1733131005458 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:18:26,330 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T09:18:26,331 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7b07d1ba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:18:26,332 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:18:26,332 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:18:26,332 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:18:26,332 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/hadoop.log.dir/,STOPPED} 2024-12-02T09:18:26,333 WARN [BP-986440887-172.17.0.3-1733131005458 heartbeating to localhost/127.0.0.1:37857 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T09:18:26,333 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T09:18:26,333 WARN [BP-986440887-172.17.0.3-1733131005458 heartbeating to localhost/127.0.0.1:37857 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-986440887-172.17.0.3-1733131005458 (Datanode Uuid 6752efd9-c80c-4b0b-b968-4b06f5453def) service to localhost/127.0.0.1:37857 2024-12-02T09:18:26,333 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T09:18:26,334 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/cluster_900e3e1e-6d1a-4bb5-1570-6ffe3d06df03/data/data1/current/BP-986440887-172.17.0.3-1733131005458 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:18:26,334 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/cluster_900e3e1e-6d1a-4bb5-1570-6ffe3d06df03/data/data2/current/BP-986440887-172.17.0.3-1733131005458 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:18:26,335 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T09:18:26,346 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@735fa16a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T09:18:26,347 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:18:26,347 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:18:26,347 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:18:26,347 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/hadoop.log.dir/,STOPPED} 2024-12-02T09:18:26,357 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-02T09:18:26,400 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-02T09:18:26,409 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=77 (was 12) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/7c6d666a4939:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: nioEventLoopGroup-5-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: nioEventLoopGroup-3-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool-2-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: nioEventLoopGroup-3-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37857 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: nioEventLoopGroup-3-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37857 from jenkins.hfs.0 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-4-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: nioEventLoopGroup-2-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: master/7c6d666a4939:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37857 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: LeaseRenewer:jenkins@localhost:37857 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37857 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: ForkJoinPool-2-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SSL Certificates Store Monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37857 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:37857 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: nioEventLoopGroup-5-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37857 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: master/7c6d666a4939:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@45316768 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=405 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=236 (was 337), ProcessCount=11 (was 11), AvailableMemoryMB=1602 (was 2102) 2024-12-02T09:18:26,417 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=78, OpenFileDescriptor=405, MaxFileDescriptor=1048576, SystemLoadAverage=236, ProcessCount=11, AvailableMemoryMB=1601 2024-12-02T09:18:26,418 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-02T09:18:26,418 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/hadoop.log.dir so I do NOT create it in target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5 2024-12-02T09:18:26,418 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f4bfabe0-9ea0-d83c-965e-a5461e3ea77b/hadoop.tmp.dir so I do NOT create it in target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5 2024-12-02T09:18:26,418 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/cluster_a52c1a07-ba90-c7c4-01a8-2a6f7d276602, deleteOnExit=true 2024-12-02T09:18:26,418 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-02T09:18:26,419 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/test.cache.data in system properties and HBase conf 2024-12-02T09:18:26,419 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/hadoop.tmp.dir in system properties and HBase conf 2024-12-02T09:18:26,419 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/hadoop.log.dir in system properties and HBase conf 2024-12-02T09:18:26,419 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-02T09:18:26,419 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-02T09:18:26,419 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-02T09:18:26,419 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-02T09:18:26,420 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-02T09:18:26,420 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-02T09:18:26,420 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-02T09:18:26,420 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T09:18:26,420 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-02T09:18:26,420 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-02T09:18:26,420 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T09:18:26,421 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T09:18:26,421 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-02T09:18:26,421 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/nfs.dump.dir in system properties and HBase conf 2024-12-02T09:18:26,421 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/java.io.tmpdir in system properties and HBase conf 2024-12-02T09:18:26,421 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T09:18:26,421 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-02T09:18:26,421 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-02T09:18:26,439 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-02T09:18:26,691 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:18:26,696 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:18:26,704 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:18:26,704 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:18:26,704 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T09:18:26,705 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:18:26,706 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@737d6115{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:18:26,706 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@641eaf99{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:18:26,801 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2a7d11e5{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/java.io.tmpdir/jetty-localhost-41047-hadoop-hdfs-3_4_1-tests_jar-_-any-10301469927603695153/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T09:18:26,801 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5d4bdc00{HTTP/1.1, (http/1.1)}{localhost:41047} 2024-12-02T09:18:26,801 INFO [Time-limited test {}] server.Server(415): Started @103282ms 2024-12-02T09:18:26,812 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-02T09:18:27,032 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:18:27,035 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:18:27,036 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:18:27,036 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:18:27,036 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T09:18:27,037 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b58749b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:18:27,037 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1aa9c156{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:18:27,129 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@66b9e907{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/java.io.tmpdir/jetty-localhost-43449-hadoop-hdfs-3_4_1-tests_jar-_-any-17173172559122315000/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:18:27,129 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@51f8cb4f{HTTP/1.1, (http/1.1)}{localhost:43449} 2024-12-02T09:18:27,129 INFO [Time-limited test {}] server.Server(415): Started @103610ms 2024-12-02T09:18:27,131 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T09:18:27,165 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:18:27,169 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:18:27,171 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:18:27,171 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:18:27,172 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T09:18:27,172 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6a742c1f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:18:27,173 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d5e070a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:18:27,270 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c17f9e5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/java.io.tmpdir/jetty-localhost-44345-hadoop-hdfs-3_4_1-tests_jar-_-any-6176345566680160781/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:18:27,270 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@669fbf01{HTTP/1.1, (http/1.1)}{localhost:44345} 2024-12-02T09:18:27,270 INFO [Time-limited test {}] server.Server(415): Started @103751ms 2024-12-02T09:18:27,272 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T09:18:27,784 WARN [Thread-453 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/cluster_a52c1a07-ba90-c7c4-01a8-2a6f7d276602/data/data1/current/BP-655264249-172.17.0.3-1733131106454/current, will proceed with Du for space computation calculation, 2024-12-02T09:18:27,784 WARN [Thread-454 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/cluster_a52c1a07-ba90-c7c4-01a8-2a6f7d276602/data/data2/current/BP-655264249-172.17.0.3-1733131106454/current, will proceed with Du for space computation calculation, 2024-12-02T09:18:27,801 WARN [Thread-418 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T09:18:27,804 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe70e26a2eeab469 with lease ID 0x287b9a939701a88b: Processing first storage report for DS-79ab9e16-26ac-442c-a13c-abc1ca7f6550 from datanode DatanodeRegistration(127.0.0.1:40695, datanodeUuid=a585234c-ea13-4320-90e5-2328bafc561d, infoPort=44817, infoSecurePort=0, ipcPort=36925, storageInfo=lv=-57;cid=testClusterID;nsid=1534929150;c=1733131106454) 2024-12-02T09:18:27,804 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe70e26a2eeab469 with lease ID 0x287b9a939701a88b: from storage DS-79ab9e16-26ac-442c-a13c-abc1ca7f6550 node DatanodeRegistration(127.0.0.1:40695, datanodeUuid=a585234c-ea13-4320-90e5-2328bafc561d, infoPort=44817, infoSecurePort=0, ipcPort=36925, storageInfo=lv=-57;cid=testClusterID;nsid=1534929150;c=1733131106454), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:18:27,804 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe70e26a2eeab469 with lease ID 0x287b9a939701a88b: Processing first storage report for DS-faed42c3-3ef8-4985-8541-2f9e28e41f16 from datanode DatanodeRegistration(127.0.0.1:40695, datanodeUuid=a585234c-ea13-4320-90e5-2328bafc561d, infoPort=44817, infoSecurePort=0, ipcPort=36925, storageInfo=lv=-57;cid=testClusterID;nsid=1534929150;c=1733131106454) 2024-12-02T09:18:27,804 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe70e26a2eeab469 with lease ID 0x287b9a939701a88b: from storage DS-faed42c3-3ef8-4985-8541-2f9e28e41f16 node DatanodeRegistration(127.0.0.1:40695, datanodeUuid=a585234c-ea13-4320-90e5-2328bafc561d, infoPort=44817, infoSecurePort=0, ipcPort=36925, storageInfo=lv=-57;cid=testClusterID;nsid=1534929150;c=1733131106454), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-02T09:18:27,969 WARN [Thread-465 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/cluster_a52c1a07-ba90-c7c4-01a8-2a6f7d276602/data/data3/current/BP-655264249-172.17.0.3-1733131106454/current, will proceed with Du for space computation calculation, 2024-12-02T09:18:27,969 WARN [Thread-466 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/cluster_a52c1a07-ba90-c7c4-01a8-2a6f7d276602/data/data4/current/BP-655264249-172.17.0.3-1733131106454/current, will proceed with Du for space computation calculation, 2024-12-02T09:18:27,988 WARN [Thread-441 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T09:18:27,991 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8471fa615494ca3e with lease ID 0x287b9a939701a88c: Processing first storage report for DS-5a6a936f-e4ed-4475-b6d0-b9ed206d0dfa from datanode DatanodeRegistration(127.0.0.1:41909, datanodeUuid=f9692e9a-2edc-4764-abfb-da6bf75066eb, infoPort=40421, infoSecurePort=0, ipcPort=38399, storageInfo=lv=-57;cid=testClusterID;nsid=1534929150;c=1733131106454) 2024-12-02T09:18:27,991 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8471fa615494ca3e with lease ID 0x287b9a939701a88c: from storage DS-5a6a936f-e4ed-4475-b6d0-b9ed206d0dfa node DatanodeRegistration(127.0.0.1:41909, datanodeUuid=f9692e9a-2edc-4764-abfb-da6bf75066eb, infoPort=40421, infoSecurePort=0, ipcPort=38399, storageInfo=lv=-57;cid=testClusterID;nsid=1534929150;c=1733131106454), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-02T09:18:27,991 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8471fa615494ca3e with lease ID 0x287b9a939701a88c: Processing first storage report for DS-1600d16d-cbf9-4749-a0cd-8cfe9374f099 from datanode DatanodeRegistration(127.0.0.1:41909, datanodeUuid=f9692e9a-2edc-4764-abfb-da6bf75066eb, infoPort=40421, infoSecurePort=0, ipcPort=38399, storageInfo=lv=-57;cid=testClusterID;nsid=1534929150;c=1733131106454) 2024-12-02T09:18:27,991 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8471fa615494ca3e with lease ID 0x287b9a939701a88c: from storage DS-1600d16d-cbf9-4749-a0cd-8cfe9374f099 node DatanodeRegistration(127.0.0.1:41909, datanodeUuid=f9692e9a-2edc-4764-abfb-da6bf75066eb, infoPort=40421, infoSecurePort=0, ipcPort=38399, storageInfo=lv=-57;cid=testClusterID;nsid=1534929150;c=1733131106454), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:18:28,015 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5 2024-12-02T09:18:28,018 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/cluster_a52c1a07-ba90-c7c4-01a8-2a6f7d276602/zookeeper_0, clientPort=59604, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/cluster_a52c1a07-ba90-c7c4-01a8-2a6f7d276602/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/cluster_a52c1a07-ba90-c7c4-01a8-2a6f7d276602/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-02T09:18:28,019 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59604 2024-12-02T09:18:28,019 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:18:28,020 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:18:28,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40695 is added to blk_1073741825_1001 (size=7) 2024-12-02T09:18:28,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41909 is added to blk_1073741825_1001 (size=7) 2024-12-02T09:18:28,031 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1 with version=8 2024-12-02T09:18:28,032 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/hbase-staging 2024-12-02T09:18:28,034 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7c6d666a4939:0 server-side Connection retries=45 2024-12-02T09:18:28,035 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:18:28,035 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T09:18:28,035 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T09:18:28,035 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:18:28,035 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T09:18:28,035 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-02T09:18:28,035 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T09:18:28,036 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41327 2024-12-02T09:18:28,038 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41327 connecting to ZooKeeper ensemble=127.0.0.1:59604 2024-12-02T09:18:28,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:413270x0, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T09:18:28,085 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41327-0x1009a47dde60000 connected 2024-12-02T09:18:28,153 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:18:28,154 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:18:28,157 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:18:28,157 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1, hbase.cluster.distributed=false 2024-12-02T09:18:28,159 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T09:18:28,159 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41327 2024-12-02T09:18:28,160 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41327 2024-12-02T09:18:28,160 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41327 2024-12-02T09:18:28,160 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41327 2024-12-02T09:18:28,161 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41327 2024-12-02T09:18:28,179 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7c6d666a4939:0 server-side Connection retries=45 2024-12-02T09:18:28,179 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:18:28,179 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T09:18:28,179 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T09:18:28,179 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:18:28,179 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T09:18:28,179 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T09:18:28,180 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T09:18:28,180 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41065 2024-12-02T09:18:28,182 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41065 connecting to ZooKeeper ensemble=127.0.0.1:59604 2024-12-02T09:18:28,183 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:18:28,186 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:18:28,194 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:410650x0, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T09:18:28,195 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:410650x0, quorum=127.0.0.1:59604, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:18:28,195 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41065-0x1009a47dde60001 connected 2024-12-02T09:18:28,195 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T09:18:28,196 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T09:18:28,197 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41065-0x1009a47dde60001, quorum=127.0.0.1:59604, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T09:18:28,198 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41065-0x1009a47dde60001, quorum=127.0.0.1:59604, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T09:18:28,198 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41065 2024-12-02T09:18:28,198 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41065 2024-12-02T09:18:28,201 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41065 2024-12-02T09:18:28,203 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41065 2024-12-02T09:18:28,204 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41065 2024-12-02T09:18:28,215 DEBUG [M:0;7c6d666a4939:41327 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7c6d666a4939:41327 2024-12-02T09:18:28,216 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7c6d666a4939,41327,1733131108034 2024-12-02T09:18:28,225 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41065-0x1009a47dde60001, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:18:28,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:18:28,226 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7c6d666a4939,41327,1733131108034 2024-12-02T09:18:28,236 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41065-0x1009a47dde60001, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T09:18:28,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:28,236 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41065-0x1009a47dde60001, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:28,236 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-02T09:18:28,237 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7c6d666a4939,41327,1733131108034 from backup master directory 2024-12-02T09:18:28,244 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41065-0x1009a47dde60001, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:18:28,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7c6d666a4939,41327,1733131108034 2024-12-02T09:18:28,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:18:28,244 WARN [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T09:18:28,244 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7c6d666a4939,41327,1733131108034 2024-12-02T09:18:28,249 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/hbase.id] with ID: c3917c09-1fd9-4497-b9d8-c327ec277d46 2024-12-02T09:18:28,249 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/.tmp/hbase.id 2024-12-02T09:18:28,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41909 is added to blk_1073741826_1002 (size=42) 2024-12-02T09:18:28,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40695 is added to blk_1073741826_1002 (size=42) 2024-12-02T09:18:28,257 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/.tmp/hbase.id]:[hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/hbase.id] 2024-12-02T09:18:28,272 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:18:28,272 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-02T09:18:28,274 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-12-02T09:18:28,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:28,286 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41065-0x1009a47dde60001, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:28,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41909 is added to blk_1073741827_1003 (size=196) 2024-12-02T09:18:28,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40695 is added to blk_1073741827_1003 (size=196) 2024-12-02T09:18:28,295 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T09:18:28,296 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-02T09:18:28,296 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-02T09:18:28,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41909 is added to blk_1073741828_1004 (size=1189) 2024-12-02T09:18:28,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40695 is added to blk_1073741828_1004 (size=1189) 2024-12-02T09:18:28,310 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/data/master/store 2024-12-02T09:18:28,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40695 is added to blk_1073741829_1005 (size=34) 2024-12-02T09:18:28,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41909 is added to blk_1073741829_1005 (size=34) 2024-12-02T09:18:28,320 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:18:28,320 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T09:18:28,320 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:18:28,320 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:18:28,320 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T09:18:28,320 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:18:28,320 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:18:28,321 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733131108320Disabling compacts and flushes for region at 1733131108320Disabling writes for close at 1733131108320Writing region close event to WAL at 1733131108320Closed at 1733131108320 2024-12-02T09:18:28,322 WARN [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/data/master/store/.initializing 2024-12-02T09:18:28,322 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/WALs/7c6d666a4939,41327,1733131108034 2024-12-02T09:18:28,325 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c6d666a4939%2C41327%2C1733131108034, suffix=, logDir=hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/WALs/7c6d666a4939,41327,1733131108034, archiveDir=hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/oldWALs, maxLogs=10 2024-12-02T09:18:28,326 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C41327%2C1733131108034.1733131108326 2024-12-02T09:18:28,332 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/WALs/7c6d666a4939,41327,1733131108034/7c6d666a4939%2C41327%2C1733131108034.1733131108326 2024-12-02T09:18:28,333 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40421:40421),(127.0.0.1/127.0.0.1:44817:44817)] 2024-12-02T09:18:28,334 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-02T09:18:28,334 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:18:28,334 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:18:28,334 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:18:28,336 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:18:28,338 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-02T09:18:28,338 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:28,338 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:18:28,339 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:18:28,340 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-02T09:18:28,340 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:28,341 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:18:28,341 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:18:28,343 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-02T09:18:28,344 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:28,344 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:18:28,344 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:18:28,346 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-02T09:18:28,346 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:28,346 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:18:28,347 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:18:28,348 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:18:28,348 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:18:28,350 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:18:28,350 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:18:28,351 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T09:18:28,352 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:18:28,355 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T09:18:28,356 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=804272, jitterRate=0.022685736417770386}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T09:18:28,357 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733131108334Initializing all the Stores at 1733131108335 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131108336 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131108336Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131108336Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131108336Cleaning up temporary data from old regions at 1733131108350 (+14 ms)Region opened successfully at 1733131108357 (+7 ms) 2024-12-02T09:18:28,358 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-02T09:18:28,363 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a723ffa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7c6d666a4939/172.17.0.3:0 2024-12-02T09:18:28,364 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-02T09:18:28,365 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-02T09:18:28,365 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-02T09:18:28,365 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T09:18:28,366 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T09:18:28,366 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-02T09:18:28,366 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-02T09:18:28,369 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-02T09:18:28,371 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-02T09:18:28,377 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-02T09:18:28,378 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-02T09:18:28,379 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-02T09:18:28,386 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-02T09:18:28,386 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-02T09:18:28,387 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-02T09:18:28,394 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-02T09:18:28,395 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-02T09:18:28,402 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-02T09:18:28,405 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-02T09:18:28,411 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-02T09:18:28,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T09:18:28,419 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41065-0x1009a47dde60001, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T09:18:28,419 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41065-0x1009a47dde60001, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:28,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:28,420 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7c6d666a4939,41327,1733131108034, sessionid=0x1009a47dde60000, setting cluster-up flag (Was=false) 2024-12-02T09:18:28,436 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41065-0x1009a47dde60001, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:28,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:28,461 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-02T09:18:28,464 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7c6d666a4939,41327,1733131108034 2024-12-02T09:18:28,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:28,486 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41065-0x1009a47dde60001, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:28,511 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-02T09:18:28,514 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7c6d666a4939,41327,1733131108034 2024-12-02T09:18:28,516 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-02T09:18:28,519 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T09:18:28,519 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-02T09:18:28,519 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-02T09:18:28,520 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7c6d666a4939,41327,1733131108034 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-02T09:18:28,522 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:18:28,522 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:18:28,522 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:18:28,522 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:18:28,522 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7c6d666a4939:0, corePoolSize=10, maxPoolSize=10 2024-12-02T09:18:28,522 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:28,522 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7c6d666a4939:0, corePoolSize=2, maxPoolSize=2 2024-12-02T09:18:28,522 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:28,523 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733131138523 2024-12-02T09:18:28,523 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-02T09:18:28,523 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-02T09:18:28,523 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-02T09:18:28,523 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-02T09:18:28,523 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-02T09:18:28,523 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-02T09:18:28,524 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:28,524 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-02T09:18:28,524 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-02T09:18:28,524 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-02T09:18:28,524 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T09:18:28,524 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-02T09:18:28,524 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-02T09:18:28,524 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-02T09:18:28,525 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.large.0-1733131108525,5,FailOnTimeoutGroup] 2024-12-02T09:18:28,525 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.small.0-1733131108525,5,FailOnTimeoutGroup] 2024-12-02T09:18:28,525 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:28,525 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-02T09:18:28,525 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:28,525 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:28,526 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:28,526 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-02T09:18:28,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41909 is added to blk_1073741831_1007 (size=1321) 2024-12-02T09:18:28,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40695 is added to blk_1073741831_1007 (size=1321) 2024-12-02T09:18:28,534 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-02T09:18:28,535 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1 2024-12-02T09:18:28,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40695 is added to blk_1073741832_1008 (size=32) 2024-12-02T09:18:28,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41909 is added to blk_1073741832_1008 (size=32) 2024-12-02T09:18:28,542 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:18:28,545 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T09:18:28,547 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T09:18:28,547 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:28,548 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:18:28,548 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T09:18:28,550 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T09:18:28,550 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:28,550 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:18:28,551 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T09:18:28,553 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T09:18:28,553 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:28,554 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:18:28,554 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T09:18:28,556 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T09:18:28,556 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:28,557 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:18:28,557 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T09:18:28,558 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/data/hbase/meta/1588230740 2024-12-02T09:18:28,558 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/data/hbase/meta/1588230740 2024-12-02T09:18:28,559 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T09:18:28,559 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T09:18:28,560 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T09:18:28,561 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T09:18:28,563 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T09:18:28,564 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=821297, jitterRate=0.04433353245258331}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T09:18:28,565 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733131108542Initializing all the Stores at 1733131108544 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131108544Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131108545 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131108545Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131108545Cleaning up temporary data from old regions at 1733131108559 (+14 ms)Region opened successfully at 1733131108565 (+6 ms) 2024-12-02T09:18:28,565 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T09:18:28,565 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T09:18:28,565 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T09:18:28,565 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T09:18:28,565 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T09:18:28,566 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T09:18:28,566 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733131108565Disabling compacts and flushes for region at 1733131108565Disabling writes for close at 1733131108565Writing region close event to WAL at 1733131108566 (+1 ms)Closed at 1733131108566 2024-12-02T09:18:28,567 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T09:18:28,567 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-02T09:18:28,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-02T09:18:28,569 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T09:18:28,571 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-02T09:18:28,606 INFO [RS:0;7c6d666a4939:41065 {}] regionserver.HRegionServer(746): ClusterId : c3917c09-1fd9-4497-b9d8-c327ec277d46 2024-12-02T09:18:28,607 DEBUG [RS:0;7c6d666a4939:41065 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T09:18:28,618 DEBUG [RS:0;7c6d666a4939:41065 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T09:18:28,618 DEBUG [RS:0;7c6d666a4939:41065 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T09:18:28,629 DEBUG [RS:0;7c6d666a4939:41065 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T09:18:28,629 DEBUG [RS:0;7c6d666a4939:41065 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16afaf9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7c6d666a4939/172.17.0.3:0 2024-12-02T09:18:28,644 DEBUG [RS:0;7c6d666a4939:41065 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7c6d666a4939:41065 2024-12-02T09:18:28,644 INFO [RS:0;7c6d666a4939:41065 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T09:18:28,644 INFO [RS:0;7c6d666a4939:41065 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T09:18:28,644 DEBUG [RS:0;7c6d666a4939:41065 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T09:18:28,645 INFO [RS:0;7c6d666a4939:41065 {}] regionserver.HRegionServer(2659): reportForDuty to master=7c6d666a4939,41327,1733131108034 with port=41065, startcode=1733131108178 2024-12-02T09:18:28,646 DEBUG [RS:0;7c6d666a4939:41065 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T09:18:28,648 INFO [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36911, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T09:18:28,649 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41327 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7c6d666a4939,41065,1733131108178 2024-12-02T09:18:28,649 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41327 {}] master.ServerManager(517): Registering regionserver=7c6d666a4939,41065,1733131108178 2024-12-02T09:18:28,651 DEBUG [RS:0;7c6d666a4939:41065 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1 2024-12-02T09:18:28,652 DEBUG [RS:0;7c6d666a4939:41065 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41123 2024-12-02T09:18:28,652 DEBUG [RS:0;7c6d666a4939:41065 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T09:18:28,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T09:18:28,661 DEBUG [RS:0;7c6d666a4939:41065 {}] zookeeper.ZKUtil(111): regionserver:41065-0x1009a47dde60001, quorum=127.0.0.1:59604, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7c6d666a4939,41065,1733131108178 2024-12-02T09:18:28,661 WARN [RS:0;7c6d666a4939:41065 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T09:18:28,661 INFO [RS:0;7c6d666a4939:41065 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-02T09:18:28,662 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7c6d666a4939,41065,1733131108178] 2024-12-02T09:18:28,662 DEBUG [RS:0;7c6d666a4939:41065 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/WALs/7c6d666a4939,41065,1733131108178 2024-12-02T09:18:28,665 INFO [RS:0;7c6d666a4939:41065 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T09:18:28,667 INFO [RS:0;7c6d666a4939:41065 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T09:18:28,668 INFO [RS:0;7c6d666a4939:41065 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T09:18:28,668 INFO [RS:0;7c6d666a4939:41065 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:28,668 INFO [RS:0;7c6d666a4939:41065 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T09:18:28,669 INFO [RS:0;7c6d666a4939:41065 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T09:18:28,669 INFO [RS:0;7c6d666a4939:41065 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:28,669 DEBUG [RS:0;7c6d666a4939:41065 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:28,669 DEBUG [RS:0;7c6d666a4939:41065 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:28,669 DEBUG [RS:0;7c6d666a4939:41065 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:28,669 DEBUG [RS:0;7c6d666a4939:41065 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:28,669 DEBUG [RS:0;7c6d666a4939:41065 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:28,669 DEBUG [RS:0;7c6d666a4939:41065 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7c6d666a4939:0, corePoolSize=2, maxPoolSize=2 2024-12-02T09:18:28,669 DEBUG [RS:0;7c6d666a4939:41065 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:28,669 DEBUG [RS:0;7c6d666a4939:41065 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:28,670 DEBUG [RS:0;7c6d666a4939:41065 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:28,670 DEBUG [RS:0;7c6d666a4939:41065 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:28,670 DEBUG [RS:0;7c6d666a4939:41065 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:28,670 DEBUG [RS:0;7c6d666a4939:41065 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:28,670 DEBUG [RS:0;7c6d666a4939:41065 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7c6d666a4939:0, corePoolSize=3, maxPoolSize=3 2024-12-02T09:18:28,670 DEBUG [RS:0;7c6d666a4939:41065 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0, corePoolSize=3, maxPoolSize=3 2024-12-02T09:18:28,670 INFO [RS:0;7c6d666a4939:41065 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:28,670 INFO [RS:0;7c6d666a4939:41065 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:28,670 INFO [RS:0;7c6d666a4939:41065 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:28,670 INFO [RS:0;7c6d666a4939:41065 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:28,670 INFO [RS:0;7c6d666a4939:41065 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:28,670 INFO [RS:0;7c6d666a4939:41065 {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,41065,1733131108178-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T09:18:28,686 INFO [RS:0;7c6d666a4939:41065 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T09:18:28,687 INFO [RS:0;7c6d666a4939:41065 {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,41065,1733131108178-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:28,687 INFO [RS:0;7c6d666a4939:41065 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:28,687 INFO [RS:0;7c6d666a4939:41065 {}] regionserver.Replication(171): 7c6d666a4939,41065,1733131108178 started 2024-12-02T09:18:28,699 INFO [RS:0;7c6d666a4939:41065 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:28,699 INFO [RS:0;7c6d666a4939:41065 {}] regionserver.HRegionServer(1482): Serving as 7c6d666a4939,41065,1733131108178, RpcServer on 7c6d666a4939/172.17.0.3:41065, sessionid=0x1009a47dde60001 2024-12-02T09:18:28,699 DEBUG [RS:0;7c6d666a4939:41065 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T09:18:28,699 DEBUG [RS:0;7c6d666a4939:41065 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7c6d666a4939,41065,1733131108178 2024-12-02T09:18:28,699 DEBUG [RS:0;7c6d666a4939:41065 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7c6d666a4939,41065,1733131108178' 2024-12-02T09:18:28,700 DEBUG [RS:0;7c6d666a4939:41065 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T09:18:28,700 DEBUG [RS:0;7c6d666a4939:41065 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T09:18:28,701 DEBUG [RS:0;7c6d666a4939:41065 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T09:18:28,701 DEBUG [RS:0;7c6d666a4939:41065 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T09:18:28,701 DEBUG [RS:0;7c6d666a4939:41065 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7c6d666a4939,41065,1733131108178 2024-12-02T09:18:28,701 DEBUG [RS:0;7c6d666a4939:41065 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7c6d666a4939,41065,1733131108178' 2024-12-02T09:18:28,701 DEBUG [RS:0;7c6d666a4939:41065 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T09:18:28,702 DEBUG [RS:0;7c6d666a4939:41065 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T09:18:28,702 DEBUG [RS:0;7c6d666a4939:41065 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T09:18:28,702 INFO [RS:0;7c6d666a4939:41065 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T09:18:28,702 INFO [RS:0;7c6d666a4939:41065 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T09:18:28,721 WARN [7c6d666a4939:41327 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-02T09:18:28,807 INFO [RS:0;7c6d666a4939:41065 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c6d666a4939%2C41065%2C1733131108178, suffix=, logDir=hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/WALs/7c6d666a4939,41065,1733131108178, archiveDir=hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/oldWALs, maxLogs=32 2024-12-02T09:18:28,812 INFO [RS:0;7c6d666a4939:41065 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C41065%2C1733131108178.1733131108812 2024-12-02T09:18:28,821 INFO [RS:0;7c6d666a4939:41065 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/WALs/7c6d666a4939,41065,1733131108178/7c6d666a4939%2C41065%2C1733131108178.1733131108812 2024-12-02T09:18:28,824 DEBUG [RS:0;7c6d666a4939:41065 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44817:44817),(127.0.0.1/127.0.0.1:40421:40421)] 2024-12-02T09:18:28,972 DEBUG [7c6d666a4939:41327 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-02T09:18:28,973 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7c6d666a4939,41065,1733131108178 2024-12-02T09:18:28,978 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7c6d666a4939,41065,1733131108178, state=OPENING 2024-12-02T09:18:28,986 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-02T09:18:28,994 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41065-0x1009a47dde60001, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:28,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:28,995 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:18:28,995 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T09:18:28,995 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:18:28,995 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7c6d666a4939,41065,1733131108178}] 2024-12-02T09:18:29,149 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T09:18:29,152 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35595, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T09:18:29,157 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-02T09:18:29,157 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-02T09:18:29,159 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c6d666a4939%2C41065%2C1733131108178.meta, suffix=.meta, logDir=hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/WALs/7c6d666a4939,41065,1733131108178, archiveDir=hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/oldWALs, maxLogs=32 2024-12-02T09:18:29,161 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C41065%2C1733131108178.meta.1733131109161.meta 2024-12-02T09:18:29,167 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/WALs/7c6d666a4939,41065,1733131108178/7c6d666a4939%2C41065%2C1733131108178.meta.1733131109161.meta 2024-12-02T09:18:29,172 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44817:44817),(127.0.0.1/127.0.0.1:40421:40421)] 2024-12-02T09:18:29,174 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-02T09:18:29,174 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-02T09:18:29,174 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-02T09:18:29,174 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-02T09:18:29,174 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-02T09:18:29,175 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:18:29,175 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-02T09:18:29,175 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-02T09:18:29,179 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T09:18:29,181 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T09:18:29,181 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:29,182 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:18:29,182 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T09:18:29,183 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T09:18:29,184 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:29,184 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:18:29,185 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T09:18:29,186 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T09:18:29,186 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:29,187 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:18:29,187 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T09:18:29,188 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T09:18:29,188 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:29,189 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:18:29,189 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T09:18:29,190 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/data/hbase/meta/1588230740 2024-12-02T09:18:29,191 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/data/hbase/meta/1588230740 2024-12-02T09:18:29,193 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T09:18:29,193 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T09:18:29,193 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T09:18:29,195 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T09:18:29,196 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=823747, jitterRate=0.04744909703731537}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T09:18:29,197 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-02T09:18:29,198 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733131109175Writing region info on filesystem at 1733131109175Initializing all the Stores at 1733131109176 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131109176Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131109179 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131109179Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131109179Cleaning up temporary data from old regions at 1733131109193 (+14 ms)Running coprocessor post-open hooks at 1733131109197 (+4 ms)Region opened successfully at 1733131109198 (+1 ms) 2024-12-02T09:18:29,199 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733131109148 2024-12-02T09:18:29,203 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-02T09:18:29,203 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-02T09:18:29,204 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7c6d666a4939,41065,1733131108178 2024-12-02T09:18:29,206 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7c6d666a4939,41065,1733131108178, state=OPEN 2024-12-02T09:18:29,236 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41065-0x1009a47dde60001, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T09:18:29,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T09:18:29,236 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7c6d666a4939,41065,1733131108178 2024-12-02T09:18:29,236 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:18:29,236 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:18:29,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-02T09:18:29,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7c6d666a4939,41065,1733131108178 in 241 msec 2024-12-02T09:18:29,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-02T09:18:29,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 673 msec 2024-12-02T09:18:29,245 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T09:18:29,245 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-02T09:18:29,247 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T09:18:29,247 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c6d666a4939,41065,1733131108178, seqNum=-1] 2024-12-02T09:18:29,247 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T09:18:29,249 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49299, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T09:18:29,258 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 738 msec 2024-12-02T09:18:29,258 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733131109258, completionTime=-1 2024-12-02T09:18:29,258 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-02T09:18:29,258 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T09:18:29,261 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-02T09:18:29,261 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733131169261 2024-12-02T09:18:29,261 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733131229261 2024-12-02T09:18:29,261 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-12-02T09:18:29,261 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,41327,1733131108034-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:29,261 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,41327,1733131108034-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:29,262 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,41327,1733131108034-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:29,262 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7c6d666a4939:41327, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:29,262 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:29,262 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:29,264 DEBUG [master/7c6d666a4939:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-02T09:18:29,267 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.023sec 2024-12-02T09:18:29,267 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-02T09:18:29,267 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-02T09:18:29,267 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-02T09:18:29,267 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-02T09:18:29,267 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-02T09:18:29,267 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,41327,1733131108034-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T09:18:29,268 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,41327,1733131108034-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-02T09:18:29,271 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-02T09:18:29,271 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-02T09:18:29,271 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,41327,1733131108034-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:29,306 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9379ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T09:18:29,306 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c6d666a4939,41327,-1 for getting cluster id 2024-12-02T09:18:29,307 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T09:18:29,308 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c3917c09-1fd9-4497-b9d8-c327ec277d46' 2024-12-02T09:18:29,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T09:18:29,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c3917c09-1fd9-4497-b9d8-c327ec277d46" 2024-12-02T09:18:29,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1288c6d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T09:18:29,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c6d666a4939,41327,-1] 2024-12-02T09:18:29,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T09:18:29,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:18:29,311 INFO [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T09:18:29,312 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aadf114, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T09:18:29,313 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T09:18:29,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c6d666a4939,41065,1733131108178, seqNum=-1] 2024-12-02T09:18:29,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T09:18:29,317 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T09:18:29,319 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7c6d666a4939,41327,1733131108034 2024-12-02T09:18:29,320 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:18:29,323 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-12-02T09:18:29,324 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-02T09:18:29,324 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T09:18:29,324 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T09:18:29,324 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:18:29,324 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:18:29,324 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T09:18:29,324 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-02T09:18:29,324 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1035653102, stopped=false 2024-12-02T09:18:29,324 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7c6d666a4939,41327,1733131108034 2024-12-02T09:18:29,336 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41065-0x1009a47dde60001, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T09:18:29,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T09:18:29,336 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T09:18:29,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:29,336 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41065-0x1009a47dde60001, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:29,336 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T09:18:29,336 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T09:18:29,337 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:18:29,337 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:18:29,337 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7c6d666a4939,41065,1733131108178' ***** 2024-12-02T09:18:29,337 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41065-0x1009a47dde60001, quorum=127.0.0.1:59604, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:18:29,337 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-02T09:18:29,337 INFO [RS:0;7c6d666a4939:41065 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T09:18:29,337 INFO [RS:0;7c6d666a4939:41065 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-02T09:18:29,337 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T09:18:29,338 INFO [RS:0;7c6d666a4939:41065 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-02T09:18:29,338 INFO [RS:0;7c6d666a4939:41065 {}] regionserver.HRegionServer(959): stopping server 7c6d666a4939,41065,1733131108178 2024-12-02T09:18:29,338 INFO [RS:0;7c6d666a4939:41065 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T09:18:29,338 INFO [RS:0;7c6d666a4939:41065 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7c6d666a4939:41065. 2024-12-02T09:18:29,338 DEBUG [RS:0;7c6d666a4939:41065 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T09:18:29,338 DEBUG [RS:0;7c6d666a4939:41065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:18:29,338 INFO [RS:0;7c6d666a4939:41065 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T09:18:29,338 INFO [RS:0;7c6d666a4939:41065 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T09:18:29,338 INFO [RS:0;7c6d666a4939:41065 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T09:18:29,338 INFO [RS:0;7c6d666a4939:41065 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-02T09:18:29,338 INFO [RS:0;7c6d666a4939:41065 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-02T09:18:29,339 DEBUG [RS:0;7c6d666a4939:41065 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-02T09:18:29,339 DEBUG [RS:0;7c6d666a4939:41065 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T09:18:29,339 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T09:18:29,339 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T09:18:29,339 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T09:18:29,339 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T09:18:29,339 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T09:18:29,339 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB 2024-12-02T09:18:29,355 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/data/hbase/meta/1588230740/.tmp/ns/74b977619b52449c84db989e1fa30856 is 43, key is default/ns:d/1733131109250/Put/seqid=0 2024-12-02T09:18:29,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40695 is added to blk_1073741835_1011 (size=5153) 2024-12-02T09:18:29,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41909 is added to blk_1073741835_1011 (size=5153) 2024-12-02T09:18:29,366 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/data/hbase/meta/1588230740/.tmp/ns/74b977619b52449c84db989e1fa30856 2024-12-02T09:18:29,374 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/data/hbase/meta/1588230740/.tmp/ns/74b977619b52449c84db989e1fa30856 as hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/data/hbase/meta/1588230740/ns/74b977619b52449c84db989e1fa30856 2024-12-02T09:18:29,382 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/data/hbase/meta/1588230740/ns/74b977619b52449c84db989e1fa30856, entries=2, sequenceid=6, filesize=5.0 K 2024-12-02T09:18:29,383 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 44ms, sequenceid=6, compaction requested=false 2024-12-02T09:18:29,383 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-02T09:18:29,388 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-02T09:18:29,389 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T09:18:29,389 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T09:18:29,389 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733131109339Running coprocessor pre-close hooks at 1733131109339Disabling compacts and flushes for region at 1733131109339Disabling writes for close at 1733131109339Obtaining lock to block concurrent updates at 1733131109339Preparing flush snapshotting stores in 1588230740 at 1733131109339Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1733131109339Flushing stores of hbase:meta,,1.1588230740 at 1733131109340 (+1 ms)Flushing 1588230740/ns: creating writer at 1733131109340Flushing 1588230740/ns: appending metadata at 1733131109354 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733131109354Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7781076e: reopening flushed file at 1733131109372 (+18 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 44ms, sequenceid=6, compaction requested=false at 1733131109383 (+11 ms)Writing region close event to WAL at 1733131109385 (+2 ms)Running coprocessor post-close hooks at 1733131109389 (+4 ms)Closed at 1733131109389 2024-12-02T09:18:29,390 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-02T09:18:29,539 INFO [RS:0;7c6d666a4939:41065 {}] regionserver.HRegionServer(976): stopping server 7c6d666a4939,41065,1733131108178; all regions closed. 2024-12-02T09:18:29,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:29,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:29,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:29,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:29,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:29,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41909 is added to blk_1073741834_1010 (size=1152) 2024-12-02T09:18:29,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40695 is added to blk_1073741834_1010 (size=1152) 2024-12-02T09:18:29,545 DEBUG [RS:0;7c6d666a4939:41065 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/oldWALs 2024-12-02T09:18:29,546 INFO [RS:0;7c6d666a4939:41065 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7c6d666a4939%2C41065%2C1733131108178.meta:.meta(num 1733131109161) 2024-12-02T09:18:29,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:29,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:29,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:29,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:29,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:29,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41909 is added to blk_1073741833_1009 (size=93) 2024-12-02T09:18:29,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40695 is added to blk_1073741833_1009 (size=93) 2024-12-02T09:18:29,551 DEBUG [RS:0;7c6d666a4939:41065 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/oldWALs 2024-12-02T09:18:29,551 INFO [RS:0;7c6d666a4939:41065 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7c6d666a4939%2C41065%2C1733131108178:(num 1733131108812) 2024-12-02T09:18:29,551 DEBUG [RS:0;7c6d666a4939:41065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:18:29,551 INFO [RS:0;7c6d666a4939:41065 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T09:18:29,551 INFO [RS:0;7c6d666a4939:41065 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T09:18:29,551 INFO [RS:0;7c6d666a4939:41065 {}] hbase.ChoreService(370): Chore service for: regionserver/7c6d666a4939:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-02T09:18:29,551 INFO [RS:0;7c6d666a4939:41065 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T09:18:29,551 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T09:18:29,552 INFO [RS:0;7c6d666a4939:41065 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41065 2024-12-02T09:18:29,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T09:18:29,561 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41065-0x1009a47dde60001, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7c6d666a4939,41065,1733131108178 2024-12-02T09:18:29,561 INFO [RS:0;7c6d666a4939:41065 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T09:18:29,569 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7c6d666a4939,41065,1733131108178] 2024-12-02T09:18:29,577 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7c6d666a4939,41065,1733131108178 already deleted, retry=false 2024-12-02T09:18:29,577 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7c6d666a4939,41065,1733131108178 expired; onlineServers=0 2024-12-02T09:18:29,577 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7c6d666a4939,41327,1733131108034' ***** 2024-12-02T09:18:29,577 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-02T09:18:29,577 INFO [M:0;7c6d666a4939:41327 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T09:18:29,578 INFO [M:0;7c6d666a4939:41327 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T09:18:29,578 DEBUG [M:0;7c6d666a4939:41327 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-02T09:18:29,578 DEBUG [M:0;7c6d666a4939:41327 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-02T09:18:29,578 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-02T09:18:29,578 DEBUG [master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.small.0-1733131108525 {}] cleaner.HFileCleaner(306): Exit Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.small.0-1733131108525,5,FailOnTimeoutGroup] 2024-12-02T09:18:29,578 DEBUG [master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.large.0-1733131108525 {}] cleaner.HFileCleaner(306): Exit Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.large.0-1733131108525,5,FailOnTimeoutGroup] 2024-12-02T09:18:29,578 INFO [M:0;7c6d666a4939:41327 {}] hbase.ChoreService(370): Chore service for: master/7c6d666a4939:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-02T09:18:29,578 INFO [M:0;7c6d666a4939:41327 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T09:18:29,578 DEBUG [M:0;7c6d666a4939:41327 {}] master.HMaster(1795): Stopping service threads 2024-12-02T09:18:29,578 INFO [M:0;7c6d666a4939:41327 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-02T09:18:29,578 INFO [M:0;7c6d666a4939:41327 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T09:18:29,579 INFO [M:0;7c6d666a4939:41327 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T09:18:29,579 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-02T09:18:29,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-02T09:18:29,586 DEBUG [M:0;7c6d666a4939:41327 {}] zookeeper.ZKUtil(347): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-02T09:18:29,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:29,586 WARN [M:0;7c6d666a4939:41327 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-02T09:18:29,587 INFO [M:0;7c6d666a4939:41327 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/.lastflushedseqids 2024-12-02T09:18:29,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41909 is added to blk_1073741836_1012 (size=99) 2024-12-02T09:18:29,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40695 is added to blk_1073741836_1012 (size=99) 2024-12-02T09:18:29,593 INFO [M:0;7c6d666a4939:41327 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T09:18:29,593 INFO [M:0;7c6d666a4939:41327 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-02T09:18:29,593 DEBUG [M:0;7c6d666a4939:41327 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T09:18:29,593 INFO [M:0;7c6d666a4939:41327 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:18:29,593 DEBUG [M:0;7c6d666a4939:41327 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:18:29,593 DEBUG [M:0;7c6d666a4939:41327 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T09:18:29,593 DEBUG [M:0;7c6d666a4939:41327 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:18:29,593 INFO [M:0;7c6d666a4939:41327 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB 2024-12-02T09:18:29,618 DEBUG [M:0;7c6d666a4939:41327 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dd8e5b8de93744e99f96dc1409813581 is 82, key is hbase:meta,,1/info:regioninfo/1733131109204/Put/seqid=0 2024-12-02T09:18:29,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40695 is added to blk_1073741837_1013 (size=5672) 2024-12-02T09:18:29,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41909 is added to blk_1073741837_1013 (size=5672) 2024-12-02T09:18:29,669 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41065-0x1009a47dde60001, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:18:29,669 INFO [RS:0;7c6d666a4939:41065 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T09:18:29,669 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41065-0x1009a47dde60001, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:18:29,669 INFO [RS:0;7c6d666a4939:41065 {}] regionserver.HRegionServer(1031): Exiting; stopping=7c6d666a4939,41065,1733131108178; zookeeper connection closed. 2024-12-02T09:18:29,670 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2975e0f5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2975e0f5 2024-12-02T09:18:29,670 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-02T09:18:29,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T09:18:29,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-02T09:18:29,835 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-12-02T09:18:30,025 INFO [M:0;7c6d666a4939:41327 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dd8e5b8de93744e99f96dc1409813581 2024-12-02T09:18:30,045 DEBUG [M:0;7c6d666a4939:41327 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e8af113f769041b195ae96df265818af is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1733131109256/Put/seqid=0 2024-12-02T09:18:30,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40695 is added to blk_1073741838_1014 (size=5275) 2024-12-02T09:18:30,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41909 is added to blk_1073741838_1014 (size=5275) 2024-12-02T09:18:30,051 INFO [M:0;7c6d666a4939:41327 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e8af113f769041b195ae96df265818af 2024-12-02T09:18:30,077 DEBUG [M:0;7c6d666a4939:41327 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a4f4d610060e42b5bec260da371070bd is 69, key is 7c6d666a4939,41065,1733131108178/rs:state/1733131108650/Put/seqid=0 2024-12-02T09:18:30,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41909 is added to blk_1073741839_1015 (size=5156) 2024-12-02T09:18:30,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40695 is added to blk_1073741839_1015 (size=5156) 2024-12-02T09:18:30,083 INFO [M:0;7c6d666a4939:41327 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a4f4d610060e42b5bec260da371070bd 2024-12-02T09:18:30,111 DEBUG [M:0;7c6d666a4939:41327 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/efc7fd375f9447cc997a57c3d29bc3f4 is 52, key is load_balancer_on/state:d/1733131109322/Put/seqid=0 2024-12-02T09:18:30,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40695 is added to blk_1073741840_1016 (size=5056) 2024-12-02T09:18:30,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41909 is added to blk_1073741840_1016 (size=5056) 2024-12-02T09:18:30,117 INFO [M:0;7c6d666a4939:41327 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/efc7fd375f9447cc997a57c3d29bc3f4 2024-12-02T09:18:30,125 DEBUG [M:0;7c6d666a4939:41327 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dd8e5b8de93744e99f96dc1409813581 as hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dd8e5b8de93744e99f96dc1409813581 2024-12-02T09:18:30,132 INFO [M:0;7c6d666a4939:41327 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dd8e5b8de93744e99f96dc1409813581, entries=8, sequenceid=29, filesize=5.5 K 2024-12-02T09:18:30,133 DEBUG [M:0;7c6d666a4939:41327 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e8af113f769041b195ae96df265818af as hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e8af113f769041b195ae96df265818af 2024-12-02T09:18:30,137 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:18:30,140 INFO [M:0;7c6d666a4939:41327 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e8af113f769041b195ae96df265818af, entries=3, sequenceid=29, filesize=5.2 K 2024-12-02T09:18:30,142 DEBUG [M:0;7c6d666a4939:41327 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a4f4d610060e42b5bec260da371070bd as hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a4f4d610060e42b5bec260da371070bd 2024-12-02T09:18:30,144 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:18:30,149 INFO [M:0;7c6d666a4939:41327 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a4f4d610060e42b5bec260da371070bd, entries=1, sequenceid=29, filesize=5.0 K 2024-12-02T09:18:30,151 DEBUG [M:0;7c6d666a4939:41327 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/efc7fd375f9447cc997a57c3d29bc3f4 as hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/efc7fd375f9447cc997a57c3d29bc3f4 2024-12-02T09:18:30,158 INFO [M:0;7c6d666a4939:41327 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41123/user/jenkins/test-data/a366b020-5330-3972-e48e-d76e51d036b1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/efc7fd375f9447cc997a57c3d29bc3f4, entries=1, sequenceid=29, filesize=4.9 K 2024-12-02T09:18:30,159 INFO [M:0;7c6d666a4939:41327 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 566ms, sequenceid=29, compaction requested=false 2024-12-02T09:18:30,163 INFO [M:0;7c6d666a4939:41327 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:18:30,163 DEBUG [M:0;7c6d666a4939:41327 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733131109593Disabling compacts and flushes for region at 1733131109593Disabling writes for close at 1733131109593Obtaining lock to block concurrent updates at 1733131109593Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733131109593Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1733131109594 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733131109598 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733131109598Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733131109617 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733131109617Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733131110031 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733131110045 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733131110045Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733131110058 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733131110077 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733131110077Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733131110090 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733131110110 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733131110110Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@690cac9: reopening flushed file at 1733131110124 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@696a6d57: reopening flushed file at 1733131110132 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a50ad1b: reopening flushed file at 1733131110140 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7658370b: reopening flushed file at 1733131110149 (+9 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 566ms, sequenceid=29, compaction requested=false at 1733131110159 (+10 ms)Writing region close event to WAL at 1733131110162 (+3 ms)Closed at 1733131110162 2024-12-02T09:18:30,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:30,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:30,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:30,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:30,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:30,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40695 is added to blk_1073741830_1006 (size=10311) 2024-12-02T09:18:30,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41909 is added to blk_1073741830_1006 (size=10311) 2024-12-02T09:18:30,167 INFO [M:0;7c6d666a4939:41327 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-02T09:18:30,167 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T09:18:30,167 INFO [M:0;7c6d666a4939:41327 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41327 2024-12-02T09:18:30,167 INFO [M:0;7c6d666a4939:41327 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T09:18:30,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:18:30,278 INFO [M:0;7c6d666a4939:41327 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T09:18:30,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41327-0x1009a47dde60000, quorum=127.0.0.1:59604, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:18:30,280 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c17f9e5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:18:30,280 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@669fbf01{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:18:30,280 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:18:30,280 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d5e070a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:18:30,280 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6a742c1f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/hadoop.log.dir/,STOPPED} 2024-12-02T09:18:30,281 WARN [BP-655264249-172.17.0.3-1733131106454 heartbeating to localhost/127.0.0.1:41123 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T09:18:30,281 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T09:18:30,281 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T09:18:30,281 WARN [BP-655264249-172.17.0.3-1733131106454 heartbeating to localhost/127.0.0.1:41123 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-655264249-172.17.0.3-1733131106454 (Datanode Uuid f9692e9a-2edc-4764-abfb-da6bf75066eb) service to localhost/127.0.0.1:41123 2024-12-02T09:18:30,282 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/cluster_a52c1a07-ba90-c7c4-01a8-2a6f7d276602/data/data3/current/BP-655264249-172.17.0.3-1733131106454 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:18:30,282 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/cluster_a52c1a07-ba90-c7c4-01a8-2a6f7d276602/data/data4/current/BP-655264249-172.17.0.3-1733131106454 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:18:30,282 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T09:18:30,284 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@66b9e907{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:18:30,285 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@51f8cb4f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:18:30,285 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:18:30,285 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1aa9c156{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:18:30,285 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7b58749b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/hadoop.log.dir/,STOPPED} 2024-12-02T09:18:30,287 WARN [BP-655264249-172.17.0.3-1733131106454 heartbeating to localhost/127.0.0.1:41123 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T09:18:30,287 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T09:18:30,287 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T09:18:30,287 WARN [BP-655264249-172.17.0.3-1733131106454 heartbeating to localhost/127.0.0.1:41123 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-655264249-172.17.0.3-1733131106454 (Datanode Uuid a585234c-ea13-4320-90e5-2328bafc561d) service to localhost/127.0.0.1:41123 2024-12-02T09:18:30,288 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/cluster_a52c1a07-ba90-c7c4-01a8-2a6f7d276602/data/data1/current/BP-655264249-172.17.0.3-1733131106454 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:18:30,288 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/cluster_a52c1a07-ba90-c7c4-01a8-2a6f7d276602/data/data2/current/BP-655264249-172.17.0.3-1733131106454 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:18:30,288 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T09:18:30,293 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2a7d11e5{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T09:18:30,293 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5d4bdc00{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:18:30,293 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:18:30,294 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@641eaf99{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:18:30,294 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@737d6115{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/hadoop.log.dir/,STOPPED} 2024-12-02T09:18:30,299 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-02T09:18:30,324 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-02T09:18:30,324 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-02T09:18:30,325 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/hadoop.log.dir so I do NOT create it in target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2 2024-12-02T09:18:30,325 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a3599dfe-d4a0-628a-342d-f645a10c2ae5/hadoop.tmp.dir so I do NOT create it in target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2 2024-12-02T09:18:30,325 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3, deleteOnExit=true 2024-12-02T09:18:30,325 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-02T09:18:30,325 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/test.cache.data in system properties and HBase conf 2024-12-02T09:18:30,325 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/hadoop.tmp.dir in system properties and HBase conf 2024-12-02T09:18:30,325 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/hadoop.log.dir in system properties and HBase conf 2024-12-02T09:18:30,326 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-02T09:18:30,326 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-02T09:18:30,326 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-02T09:18:30,326 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-02T09:18:30,326 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-02T09:18:30,326 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-02T09:18:30,327 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-02T09:18:30,327 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T09:18:30,327 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-02T09:18:30,327 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-02T09:18:30,327 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T09:18:30,327 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T09:18:30,327 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-02T09:18:30,328 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/nfs.dump.dir in system properties and HBase conf 2024-12-02T09:18:30,328 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/java.io.tmpdir in system properties and HBase conf 2024-12-02T09:18:30,328 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T09:18:30,328 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-02T09:18:30,328 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-02T09:18:30,346 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-02T09:18:30,569 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-02T09:18:30,571 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:18:30,582 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:18:30,583 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:18:30,583 WARN [Time-limited test {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:18:30,602 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:18:30,607 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:18:30,611 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:18:30,611 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:18:30,611 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T09:18:30,612 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:18:30,613 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3b918d2a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:18:30,613 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20aa2ea7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:18:30,671 INFO [regionserver/7c6d666a4939:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T09:18:30,708 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2e195dbd{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/java.io.tmpdir/jetty-localhost-43623-hadoop-hdfs-3_4_1-tests_jar-_-any-5530524745865461635/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T09:18:30,709 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5d9b1613{HTTP/1.1, (http/1.1)}{localhost:43623} 2024-12-02T09:18:30,709 INFO [Time-limited test {}] server.Server(415): Started @107189ms 2024-12-02T09:18:30,722 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-02T09:18:30,911 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:18:30,914 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:18:30,915 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:18:30,915 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:18:30,915 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T09:18:30,916 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@651aa118{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:18:30,916 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@48d478e7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:18:31,008 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3e10767c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/java.io.tmpdir/jetty-localhost-38659-hadoop-hdfs-3_4_1-tests_jar-_-any-13714487451718556737/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:18:31,008 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4d2d9832{HTTP/1.1, (http/1.1)}{localhost:38659} 2024-12-02T09:18:31,008 INFO [Time-limited test {}] server.Server(415): Started @107489ms 2024-12-02T09:18:31,010 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T09:18:31,044 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:18:31,049 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:18:31,049 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:18:31,050 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:18:31,050 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T09:18:31,050 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3d5daa57{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:18:31,051 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@10a4d310{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:18:31,144 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@544fa662{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/java.io.tmpdir/jetty-localhost-37269-hadoop-hdfs-3_4_1-tests_jar-_-any-15030478860487565365/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:18:31,144 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3574ce3f{HTTP/1.1, (http/1.1)}{localhost:37269} 2024-12-02T09:18:31,144 INFO [Time-limited test {}] server.Server(415): Started @107625ms 2024-12-02T09:18:31,145 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T09:18:31,715 WARN [Thread-675 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data2/current/BP-931683727-172.17.0.3-1733131110360/current, will proceed with Du for space computation calculation, 2024-12-02T09:18:31,715 WARN [Thread-674 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data1/current/BP-931683727-172.17.0.3-1733131110360/current, will proceed with Du for space computation calculation, 2024-12-02T09:18:31,735 WARN [Thread-638 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T09:18:31,738 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x593c08f0261fd4b9 with lease ID 0xe9c6559de3b4cea8: Processing first storage report for DS-b5c186eb-2651-4937-820a-5fbdb03c4e97 from datanode DatanodeRegistration(127.0.0.1:40489, datanodeUuid=4f2da047-ed4a-4df8-9d20-04093f4eaca4, infoPort=46879, infoSecurePort=0, ipcPort=41383, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360) 2024-12-02T09:18:31,738 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x593c08f0261fd4b9 with lease ID 0xe9c6559de3b4cea8: from storage DS-b5c186eb-2651-4937-820a-5fbdb03c4e97 node DatanodeRegistration(127.0.0.1:40489, datanodeUuid=4f2da047-ed4a-4df8-9d20-04093f4eaca4, infoPort=46879, infoSecurePort=0, ipcPort=41383, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:18:31,738 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x593c08f0261fd4b9 with lease ID 0xe9c6559de3b4cea8: Processing first storage report for DS-0ca0a1b7-f82b-4a01-9e4e-ee99e5c82470 from datanode DatanodeRegistration(127.0.0.1:40489, datanodeUuid=4f2da047-ed4a-4df8-9d20-04093f4eaca4, infoPort=46879, infoSecurePort=0, ipcPort=41383, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360) 2024-12-02T09:18:31,738 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x593c08f0261fd4b9 with lease ID 0xe9c6559de3b4cea8: from storage DS-0ca0a1b7-f82b-4a01-9e4e-ee99e5c82470 node DatanodeRegistration(127.0.0.1:40489, datanodeUuid=4f2da047-ed4a-4df8-9d20-04093f4eaca4, infoPort=46879, infoSecurePort=0, ipcPort=41383, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:18:31,868 WARN [Thread-686 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data4/current/BP-931683727-172.17.0.3-1733131110360/current, will proceed with Du for space computation calculation, 2024-12-02T09:18:31,868 WARN [Thread-685 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data3/current/BP-931683727-172.17.0.3-1733131110360/current, will proceed with Du for space computation calculation, 2024-12-02T09:18:31,890 WARN [Thread-661 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T09:18:31,892 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x97f1b9d3bf6a997c with lease ID 0xe9c6559de3b4cea9: Processing first storage report for DS-99e799c5-14e9-4935-8480-11c3a0722912 from datanode DatanodeRegistration(127.0.0.1:39375, datanodeUuid=900a00a2-9ff5-4c37-9912-5024947f217c, infoPort=45099, infoSecurePort=0, ipcPort=33571, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360) 2024-12-02T09:18:31,893 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x97f1b9d3bf6a997c with lease ID 0xe9c6559de3b4cea9: from storage DS-99e799c5-14e9-4935-8480-11c3a0722912 node DatanodeRegistration(127.0.0.1:39375, datanodeUuid=900a00a2-9ff5-4c37-9912-5024947f217c, infoPort=45099, infoSecurePort=0, ipcPort=33571, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:18:31,893 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x97f1b9d3bf6a997c with lease ID 0xe9c6559de3b4cea9: Processing first storage report for DS-4cedcc62-af50-4a9b-a537-caa0c79444bf from datanode DatanodeRegistration(127.0.0.1:39375, datanodeUuid=900a00a2-9ff5-4c37-9912-5024947f217c, infoPort=45099, infoSecurePort=0, ipcPort=33571, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360) 2024-12-02T09:18:31,893 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x97f1b9d3bf6a997c with lease ID 0xe9c6559de3b4cea9: from storage DS-4cedcc62-af50-4a9b-a537-caa0c79444bf node DatanodeRegistration(127.0.0.1:39375, datanodeUuid=900a00a2-9ff5-4c37-9912-5024947f217c, infoPort=45099, infoSecurePort=0, ipcPort=33571, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:18:31,978 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2 2024-12-02T09:18:31,981 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/zookeeper_0, clientPort=51860, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-02T09:18:31,982 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51860 2024-12-02T09:18:31,982 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:18:31,984 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:18:31,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40489 is added to blk_1073741825_1001 (size=7) 2024-12-02T09:18:31,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39375 is added to blk_1073741825_1001 (size=7) 2024-12-02T09:18:31,998 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c with version=8 2024-12-02T09:18:31,998 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/hbase-staging 2024-12-02T09:18:32,000 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7c6d666a4939:0 server-side Connection retries=45 2024-12-02T09:18:32,000 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:18:32,000 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T09:18:32,000 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T09:18:32,000 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:18:32,000 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T09:18:32,001 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-02T09:18:32,001 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T09:18:32,001 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38901 2024-12-02T09:18:32,002 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38901 connecting to ZooKeeper ensemble=127.0.0.1:51860 2024-12-02T09:18:32,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:389010x0, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T09:18:32,052 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38901-0x1009a47ed610000 connected 2024-12-02T09:18:32,119 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:18:32,121 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:18:32,124 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:18:32,125 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c, hbase.cluster.distributed=false 2024-12-02T09:18:32,127 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T09:18:32,127 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38901 2024-12-02T09:18:32,127 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38901 2024-12-02T09:18:32,128 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38901 2024-12-02T09:18:32,128 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38901 2024-12-02T09:18:32,128 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38901 2024-12-02T09:18:32,143 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7c6d666a4939:0 server-side Connection retries=45 2024-12-02T09:18:32,143 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:18:32,143 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T09:18:32,143 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T09:18:32,143 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:18:32,143 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T09:18:32,143 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T09:18:32,143 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T09:18:32,144 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37839 2024-12-02T09:18:32,146 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37839 connecting to ZooKeeper ensemble=127.0.0.1:51860 2024-12-02T09:18:32,146 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:18:32,148 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:18:32,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:378390x0, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T09:18:32,161 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:378390x0, quorum=127.0.0.1:51860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:18:32,161 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37839-0x1009a47ed610001 connected 2024-12-02T09:18:32,162 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T09:18:32,162 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T09:18:32,163 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37839-0x1009a47ed610001, quorum=127.0.0.1:51860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T09:18:32,164 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37839-0x1009a47ed610001, quorum=127.0.0.1:51860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T09:18:32,165 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37839 2024-12-02T09:18:32,165 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37839 2024-12-02T09:18:32,167 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37839 2024-12-02T09:18:32,168 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37839 2024-12-02T09:18:32,168 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37839 2024-12-02T09:18:32,182 DEBUG [M:0;7c6d666a4939:38901 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7c6d666a4939:38901 2024-12-02T09:18:32,183 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7c6d666a4939,38901,1733131112000 2024-12-02T09:18:32,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:18:32,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37839-0x1009a47ed610001, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:18:32,200 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7c6d666a4939,38901,1733131112000 2024-12-02T09:18:32,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:32,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37839-0x1009a47ed610001, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T09:18:32,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37839-0x1009a47ed610001, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:32,217 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-02T09:18:32,218 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7c6d666a4939,38901,1733131112000 from backup master directory 2024-12-02T09:18:32,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37839-0x1009a47ed610001, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:18:32,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7c6d666a4939,38901,1733131112000 2024-12-02T09:18:32,227 WARN [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T09:18:32,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:18:32,227 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7c6d666a4939,38901,1733131112000 2024-12-02T09:18:32,233 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/hbase.id] with ID: 74ee40a3-5dbd-462a-8009-8d6095d083ad 2024-12-02T09:18:32,233 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/.tmp/hbase.id 2024-12-02T09:18:32,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39375 is added to blk_1073741826_1002 (size=42) 2024-12-02T09:18:32,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40489 is added to blk_1073741826_1002 (size=42) 2024-12-02T09:18:32,243 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/.tmp/hbase.id]:[hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/hbase.id] 2024-12-02T09:18:32,258 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:18:32,258 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-02T09:18:32,259 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-12-02T09:18:32,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:32,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37839-0x1009a47ed610001, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:32,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39375 is added to blk_1073741827_1003 (size=196) 2024-12-02T09:18:32,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40489 is added to blk_1073741827_1003 (size=196) 2024-12-02T09:18:32,283 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T09:18:32,284 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-02T09:18:32,284 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-02T09:18:32,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39375 is added to blk_1073741828_1004 (size=1189) 2024-12-02T09:18:32,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40489 is added to blk_1073741828_1004 (size=1189) 2024-12-02T09:18:32,296 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/data/master/store 2024-12-02T09:18:32,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39375 is added to blk_1073741829_1005 (size=34) 2024-12-02T09:18:32,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40489 is added to blk_1073741829_1005 (size=34) 2024-12-02T09:18:32,304 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:18:32,304 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T09:18:32,304 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:18:32,304 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:18:32,304 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T09:18:32,304 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:18:32,304 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:18:32,304 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733131112304Disabling compacts and flushes for region at 1733131112304Disabling writes for close at 1733131112304Writing region close event to WAL at 1733131112304Closed at 1733131112304 2024-12-02T09:18:32,305 WARN [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/data/master/store/.initializing 2024-12-02T09:18:32,305 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/WALs/7c6d666a4939,38901,1733131112000 2024-12-02T09:18:32,308 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c6d666a4939%2C38901%2C1733131112000, suffix=, logDir=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/WALs/7c6d666a4939,38901,1733131112000, archiveDir=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/oldWALs, maxLogs=10 2024-12-02T09:18:32,309 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C38901%2C1733131112000.1733131112309 2024-12-02T09:18:32,314 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/WALs/7c6d666a4939,38901,1733131112000/7c6d666a4939%2C38901%2C1733131112000.1733131112309 2024-12-02T09:18:32,315 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46879:46879),(127.0.0.1/127.0.0.1:45099:45099)] 2024-12-02T09:18:32,315 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-02T09:18:32,316 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:18:32,316 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:18:32,316 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:18:32,317 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:18:32,319 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-02T09:18:32,319 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:32,319 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:18:32,319 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:18:32,321 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-02T09:18:32,321 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:32,321 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:18:32,322 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:18:32,323 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-02T09:18:32,323 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:32,323 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:18:32,323 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:18:32,325 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-02T09:18:32,325 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:32,325 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:18:32,326 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:18:32,326 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:18:32,327 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:18:32,328 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:18:32,329 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:18:32,329 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T09:18:32,331 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:18:32,333 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T09:18:32,334 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=713444, jitterRate=-0.09280957281589508}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T09:18:32,334 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733131112316Initializing all the Stores at 1733131112317 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131112317Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131112317Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131112317Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131112317Cleaning up temporary data from old regions at 1733131112329 (+12 ms)Region opened successfully at 1733131112334 (+5 ms) 2024-12-02T09:18:32,335 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-02T09:18:32,338 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@622e4bec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7c6d666a4939/172.17.0.3:0 2024-12-02T09:18:32,339 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-02T09:18:32,339 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-02T09:18:32,339 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-02T09:18:32,339 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T09:18:32,340 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T09:18:32,340 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-02T09:18:32,340 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-02T09:18:32,342 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-02T09:18:32,343 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-02T09:18:32,352 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-02T09:18:32,353 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-02T09:18:32,353 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-02T09:18:32,360 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-02T09:18:32,361 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-02T09:18:32,362 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-02T09:18:32,369 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-02T09:18:32,370 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-02T09:18:32,377 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-02T09:18:32,379 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-02T09:18:32,385 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-02T09:18:32,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T09:18:32,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37839-0x1009a47ed610001, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T09:18:32,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:32,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37839-0x1009a47ed610001, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:32,395 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7c6d666a4939,38901,1733131112000, sessionid=0x1009a47ed610000, setting cluster-up flag (Was=false) 2024-12-02T09:18:32,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:32,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37839-0x1009a47ed610001, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:32,444 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-02T09:18:32,445 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7c6d666a4939,38901,1733131112000 2024-12-02T09:18:32,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37839-0x1009a47ed610001, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:32,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:32,494 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-02T09:18:32,495 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7c6d666a4939,38901,1733131112000 2024-12-02T09:18:32,497 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-02T09:18:32,520 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T09:18:32,521 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-02T09:18:32,521 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-02T09:18:32,521 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7c6d666a4939,38901,1733131112000 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-02T09:18:32,523 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:18:32,523 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:18:32,523 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:18:32,523 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:18:32,523 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7c6d666a4939:0, corePoolSize=10, maxPoolSize=10 2024-12-02T09:18:32,523 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:32,523 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7c6d666a4939:0, corePoolSize=2, maxPoolSize=2 2024-12-02T09:18:32,523 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:32,524 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733131142524 2024-12-02T09:18:32,525 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-02T09:18:32,525 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-02T09:18:32,525 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-02T09:18:32,525 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-02T09:18:32,525 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-02T09:18:32,525 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-02T09:18:32,525 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:32,525 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T09:18:32,525 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-02T09:18:32,525 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-02T09:18:32,525 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-02T09:18:32,526 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-02T09:18:32,526 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-02T09:18:32,526 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-02T09:18:32,526 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.large.0-1733131112526,5,FailOnTimeoutGroup] 2024-12-02T09:18:32,526 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.small.0-1733131112526,5,FailOnTimeoutGroup] 2024-12-02T09:18:32,526 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:32,526 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-02T09:18:32,526 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:32,527 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:32,527 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:32,527 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-02T09:18:32,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40489 is added to blk_1073741831_1007 (size=1321) 2024-12-02T09:18:32,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39375 is added to blk_1073741831_1007 (size=1321) 2024-12-02T09:18:32,539 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-02T09:18:32,539 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c 2024-12-02T09:18:32,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39375 is added to blk_1073741832_1008 (size=32) 2024-12-02T09:18:32,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40489 is added to blk_1073741832_1008 (size=32) 2024-12-02T09:18:32,549 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:18:32,553 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T09:18:32,555 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T09:18:32,555 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:32,556 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:18:32,556 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T09:18:32,558 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T09:18:32,558 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:32,558 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:18:32,558 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T09:18:32,560 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T09:18:32,560 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:32,561 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:18:32,561 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T09:18:32,562 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T09:18:32,562 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:32,563 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:18:32,563 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T09:18:32,564 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/hbase/meta/1588230740 2024-12-02T09:18:32,564 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/hbase/meta/1588230740 2024-12-02T09:18:32,566 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T09:18:32,566 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T09:18:32,567 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T09:18:32,569 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T09:18:32,570 INFO [RS:0;7c6d666a4939:37839 {}] regionserver.HRegionServer(746): ClusterId : 74ee40a3-5dbd-462a-8009-8d6095d083ad 2024-12-02T09:18:32,570 DEBUG [RS:0;7c6d666a4939:37839 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T09:18:32,572 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T09:18:32,572 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=825970, jitterRate=0.050275444984436035}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T09:18:32,573 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733131112549Initializing all the Stores at 1733131112552 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131112552Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131112553 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131112553Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131112553Cleaning up temporary data from old regions at 1733131112566 (+13 ms)Region opened successfully at 1733131112573 (+7 ms) 2024-12-02T09:18:32,573 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T09:18:32,573 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T09:18:32,573 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T09:18:32,574 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T09:18:32,574 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T09:18:32,574 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T09:18:32,574 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733131112573Disabling compacts and flushes for region at 1733131112573Disabling writes for close at 1733131112574 (+1 ms)Writing region close event to WAL at 1733131112574Closed at 1733131112574 2024-12-02T09:18:32,576 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T09:18:32,576 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-02T09:18:32,576 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-02T09:18:32,578 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T09:18:32,578 DEBUG [RS:0;7c6d666a4939:37839 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T09:18:32,578 DEBUG [RS:0;7c6d666a4939:37839 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T09:18:32,579 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-02T09:18:32,586 DEBUG [RS:0;7c6d666a4939:37839 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T09:18:32,587 DEBUG [RS:0;7c6d666a4939:37839 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa8e196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7c6d666a4939/172.17.0.3:0 2024-12-02T09:18:32,603 DEBUG [RS:0;7c6d666a4939:37839 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7c6d666a4939:37839 2024-12-02T09:18:32,604 INFO [RS:0;7c6d666a4939:37839 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T09:18:32,604 INFO [RS:0;7c6d666a4939:37839 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T09:18:32,604 DEBUG [RS:0;7c6d666a4939:37839 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T09:18:32,605 INFO [RS:0;7c6d666a4939:37839 {}] regionserver.HRegionServer(2659): reportForDuty to master=7c6d666a4939,38901,1733131112000 with port=37839, startcode=1733131112142 2024-12-02T09:18:32,605 DEBUG [RS:0;7c6d666a4939:37839 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T09:18:32,607 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36031, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T09:18:32,607 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38901 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7c6d666a4939,37839,1733131112142 2024-12-02T09:18:32,608 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38901 {}] master.ServerManager(517): Registering regionserver=7c6d666a4939,37839,1733131112142 2024-12-02T09:18:32,609 DEBUG [RS:0;7c6d666a4939:37839 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c 2024-12-02T09:18:32,609 DEBUG [RS:0;7c6d666a4939:37839 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40751 2024-12-02T09:18:32,609 DEBUG [RS:0;7c6d666a4939:37839 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T09:18:32,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T09:18:32,619 DEBUG [RS:0;7c6d666a4939:37839 {}] zookeeper.ZKUtil(111): regionserver:37839-0x1009a47ed610001, quorum=127.0.0.1:51860, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7c6d666a4939,37839,1733131112142 2024-12-02T09:18:32,620 WARN [RS:0;7c6d666a4939:37839 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T09:18:32,620 INFO [RS:0;7c6d666a4939:37839 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-02T09:18:32,620 DEBUG [RS:0;7c6d666a4939:37839 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142 2024-12-02T09:18:32,620 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7c6d666a4939,37839,1733131112142] 2024-12-02T09:18:32,624 INFO [RS:0;7c6d666a4939:37839 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T09:18:32,625 INFO [RS:0;7c6d666a4939:37839 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T09:18:32,626 INFO [RS:0;7c6d666a4939:37839 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T09:18:32,626 INFO [RS:0;7c6d666a4939:37839 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:32,627 INFO [RS:0;7c6d666a4939:37839 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T09:18:32,628 INFO [RS:0;7c6d666a4939:37839 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T09:18:32,628 INFO [RS:0;7c6d666a4939:37839 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:32,628 DEBUG [RS:0;7c6d666a4939:37839 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:32,629 DEBUG [RS:0;7c6d666a4939:37839 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:32,629 DEBUG [RS:0;7c6d666a4939:37839 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:32,629 DEBUG [RS:0;7c6d666a4939:37839 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:32,629 DEBUG [RS:0;7c6d666a4939:37839 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:32,629 DEBUG [RS:0;7c6d666a4939:37839 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7c6d666a4939:0, corePoolSize=2, maxPoolSize=2 2024-12-02T09:18:32,629 DEBUG [RS:0;7c6d666a4939:37839 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:32,629 DEBUG [RS:0;7c6d666a4939:37839 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:32,629 DEBUG [RS:0;7c6d666a4939:37839 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:32,629 DEBUG [RS:0;7c6d666a4939:37839 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:32,629 DEBUG [RS:0;7c6d666a4939:37839 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:32,629 DEBUG [RS:0;7c6d666a4939:37839 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:32,629 DEBUG [RS:0;7c6d666a4939:37839 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7c6d666a4939:0, corePoolSize=3, maxPoolSize=3 2024-12-02T09:18:32,629 DEBUG [RS:0;7c6d666a4939:37839 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0, corePoolSize=3, maxPoolSize=3 2024-12-02T09:18:32,630 INFO [RS:0;7c6d666a4939:37839 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:32,630 INFO [RS:0;7c6d666a4939:37839 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:32,630 INFO [RS:0;7c6d666a4939:37839 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:32,630 INFO [RS:0;7c6d666a4939:37839 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:32,630 INFO [RS:0;7c6d666a4939:37839 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:32,630 INFO [RS:0;7c6d666a4939:37839 {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,37839,1733131112142-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T09:18:32,646 INFO [RS:0;7c6d666a4939:37839 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T09:18:32,646 INFO [RS:0;7c6d666a4939:37839 {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,37839,1733131112142-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:32,646 INFO [RS:0;7c6d666a4939:37839 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:32,646 INFO [RS:0;7c6d666a4939:37839 {}] regionserver.Replication(171): 7c6d666a4939,37839,1733131112142 started 2024-12-02T09:18:32,667 INFO [RS:0;7c6d666a4939:37839 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:32,667 INFO [RS:0;7c6d666a4939:37839 {}] regionserver.HRegionServer(1482): Serving as 7c6d666a4939,37839,1733131112142, RpcServer on 7c6d666a4939/172.17.0.3:37839, sessionid=0x1009a47ed610001 2024-12-02T09:18:32,667 DEBUG [RS:0;7c6d666a4939:37839 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T09:18:32,667 DEBUG [RS:0;7c6d666a4939:37839 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7c6d666a4939,37839,1733131112142 2024-12-02T09:18:32,667 DEBUG [RS:0;7c6d666a4939:37839 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7c6d666a4939,37839,1733131112142' 2024-12-02T09:18:32,667 DEBUG [RS:0;7c6d666a4939:37839 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T09:18:32,668 DEBUG [RS:0;7c6d666a4939:37839 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T09:18:32,669 DEBUG [RS:0;7c6d666a4939:37839 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T09:18:32,670 DEBUG [RS:0;7c6d666a4939:37839 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T09:18:32,670 DEBUG [RS:0;7c6d666a4939:37839 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7c6d666a4939,37839,1733131112142 2024-12-02T09:18:32,670 DEBUG [RS:0;7c6d666a4939:37839 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7c6d666a4939,37839,1733131112142' 2024-12-02T09:18:32,670 DEBUG [RS:0;7c6d666a4939:37839 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T09:18:32,670 DEBUG [RS:0;7c6d666a4939:37839 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T09:18:32,671 DEBUG [RS:0;7c6d666a4939:37839 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T09:18:32,671 INFO [RS:0;7c6d666a4939:37839 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T09:18:32,671 INFO [RS:0;7c6d666a4939:37839 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T09:18:32,730 WARN [7c6d666a4939:38901 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-02T09:18:32,774 INFO [RS:0;7c6d666a4939:37839 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c6d666a4939%2C37839%2C1733131112142, suffix=, logDir=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142, archiveDir=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/oldWALs, maxLogs=32 2024-12-02T09:18:32,777 INFO [RS:0;7c6d666a4939:37839 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C37839%2C1733131112142.1733131112776 2024-12-02T09:18:32,784 INFO [RS:0;7c6d666a4939:37839 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131112776 2024-12-02T09:18:32,788 DEBUG [RS:0;7c6d666a4939:37839 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46879:46879),(127.0.0.1/127.0.0.1:45099:45099)] 2024-12-02T09:18:32,980 DEBUG [7c6d666a4939:38901 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-02T09:18:32,982 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7c6d666a4939,37839,1733131112142 2024-12-02T09:18:32,985 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7c6d666a4939,37839,1733131112142, state=OPENING 2024-12-02T09:18:32,994 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-02T09:18:33,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37839-0x1009a47ed610001, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:33,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:18:33,004 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:18:33,005 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:18:33,005 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T09:18:33,005 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7c6d666a4939,37839,1733131112142}] 2024-12-02T09:18:33,159 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T09:18:33,162 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49673, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T09:18:33,171 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-02T09:18:33,171 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-02T09:18:33,174 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c6d666a4939%2C37839%2C1733131112142.meta, suffix=.meta, logDir=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142, archiveDir=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/oldWALs, maxLogs=32 2024-12-02T09:18:33,174 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta 2024-12-02T09:18:33,187 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta 2024-12-02T09:18:33,195 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46879:46879),(127.0.0.1/127.0.0.1:45099:45099)] 2024-12-02T09:18:33,197 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-02T09:18:33,197 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-02T09:18:33,198 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-02T09:18:33,198 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-02T09:18:33,198 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-02T09:18:33,198 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:18:33,198 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-02T09:18:33,198 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-02T09:18:33,200 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T09:18:33,202 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T09:18:33,202 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:33,203 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:18:33,203 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T09:18:33,205 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T09:18:33,205 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:33,206 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:18:33,206 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T09:18:33,207 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T09:18:33,207 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:33,208 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:18:33,209 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T09:18:33,210 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T09:18:33,210 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:33,211 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:18:33,211 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T09:18:33,212 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/hbase/meta/1588230740 2024-12-02T09:18:33,214 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/hbase/meta/1588230740 2024-12-02T09:18:33,216 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T09:18:33,216 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T09:18:33,217 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T09:18:33,218 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T09:18:33,220 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=792821, jitterRate=0.008124589920043945}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T09:18:33,220 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-02T09:18:33,221 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733131113199Writing region info on filesystem at 1733131113199Initializing all the Stores at 1733131113200 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131113200Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131113200Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131113200Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131113200Cleaning up temporary data from old regions at 1733131113216 (+16 ms)Running coprocessor post-open hooks at 1733131113220 (+4 ms)Region opened successfully at 1733131113221 (+1 ms) 2024-12-02T09:18:33,222 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733131113159 2024-12-02T09:18:33,227 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-02T09:18:33,227 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-02T09:18:33,228 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7c6d666a4939,37839,1733131112142 2024-12-02T09:18:33,230 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7c6d666a4939,37839,1733131112142, state=OPEN 2024-12-02T09:18:33,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T09:18:33,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37839-0x1009a47ed610001, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T09:18:33,269 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7c6d666a4939,37839,1733131112142 2024-12-02T09:18:33,269 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:18:33,269 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:18:33,272 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-02T09:18:33,273 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7c6d666a4939,37839,1733131112142 in 264 msec 2024-12-02T09:18:33,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-02T09:18:33,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 697 msec 2024-12-02T09:18:33,277 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T09:18:33,277 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-02T09:18:33,279 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T09:18:33,280 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c6d666a4939,37839,1733131112142, seqNum=-1] 2024-12-02T09:18:33,280 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T09:18:33,282 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54387, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T09:18:33,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 791 msec 2024-12-02T09:18:33,293 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733131113293, completionTime=-1 2024-12-02T09:18:33,293 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-02T09:18:33,293 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T09:18:33,296 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-02T09:18:33,296 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733131173296 2024-12-02T09:18:33,296 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733131233296 2024-12-02T09:18:33,297 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-12-02T09:18:33,297 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,38901,1733131112000-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:33,297 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,38901,1733131112000-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:33,297 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,38901,1733131112000-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:33,297 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7c6d666a4939:38901, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:33,297 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:33,299 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:33,300 DEBUG [master/7c6d666a4939:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-02T09:18:33,305 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.078sec 2024-12-02T09:18:33,305 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-02T09:18:33,305 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-02T09:18:33,305 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-02T09:18:33,305 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-02T09:18:33,305 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-02T09:18:33,305 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,38901,1733131112000-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T09:18:33,305 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,38901,1733131112000-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-02T09:18:33,308 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-02T09:18:33,308 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-02T09:18:33,308 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,38901,1733131112000-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:33,371 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@543a1dd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T09:18:33,371 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c6d666a4939,38901,-1 for getting cluster id 2024-12-02T09:18:33,372 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T09:18:33,380 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74ee40a3-5dbd-462a-8009-8d6095d083ad' 2024-12-02T09:18:33,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T09:18:33,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74ee40a3-5dbd-462a-8009-8d6095d083ad" 2024-12-02T09:18:33,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eac6a37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T09:18:33,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c6d666a4939,38901,-1] 2024-12-02T09:18:33,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T09:18:33,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:18:33,383 INFO [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T09:18:33,384 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337c5dd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T09:18:33,384 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T09:18:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c6d666a4939,37839,1733131112142, seqNum=-1] 2024-12-02T09:18:33,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T09:18:33,388 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T09:18:33,389 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7c6d666a4939,38901,1733131112000 2024-12-02T09:18:33,390 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:18:33,393 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-12-02T09:18:33,414 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7c6d666a4939:0 server-side Connection retries=45 2024-12-02T09:18:33,414 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:18:33,414 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T09:18:33,414 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T09:18:33,414 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:18:33,414 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T09:18:33,414 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T09:18:33,414 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T09:18:33,415 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39613 2024-12-02T09:18:33,416 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39613 connecting to ZooKeeper ensemble=127.0.0.1:51860 2024-12-02T09:18:33,417 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:18:33,418 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:18:33,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:396130x0, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T09:18:33,428 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:396130x0, quorum=127.0.0.1:51860, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-02T09:18:33,428 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-02T09:18:33,428 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39613-0x1009a47ed610002 connected 2024-12-02T09:18:33,429 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T09:18:33,429 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T09:18:33,430 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:39613-0x1009a47ed610002, quorum=127.0.0.1:51860, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-02T09:18:33,431 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39613-0x1009a47ed610002, quorum=127.0.0.1:51860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T09:18:33,434 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39613 2024-12-02T09:18:33,434 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39613 2024-12-02T09:18:33,435 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39613 2024-12-02T09:18:33,435 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39613 2024-12-02T09:18:33,435 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39613 2024-12-02T09:18:33,437 INFO [RS:1;7c6d666a4939:39613 {}] regionserver.HRegionServer(746): ClusterId : 74ee40a3-5dbd-462a-8009-8d6095d083ad 2024-12-02T09:18:33,437 DEBUG [RS:1;7c6d666a4939:39613 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T09:18:33,445 DEBUG [RS:1;7c6d666a4939:39613 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T09:18:33,445 DEBUG [RS:1;7c6d666a4939:39613 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T09:18:33,453 DEBUG [RS:1;7c6d666a4939:39613 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T09:18:33,454 DEBUG [RS:1;7c6d666a4939:39613 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55f5caa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7c6d666a4939/172.17.0.3:0 2024-12-02T09:18:33,466 DEBUG [RS:1;7c6d666a4939:39613 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;7c6d666a4939:39613 2024-12-02T09:18:33,466 INFO [RS:1;7c6d666a4939:39613 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T09:18:33,466 INFO [RS:1;7c6d666a4939:39613 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T09:18:33,466 DEBUG [RS:1;7c6d666a4939:39613 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T09:18:33,467 INFO [RS:1;7c6d666a4939:39613 {}] regionserver.HRegionServer(2659): reportForDuty to master=7c6d666a4939,38901,1733131112000 with port=39613, startcode=1733131113414 2024-12-02T09:18:33,467 DEBUG [RS:1;7c6d666a4939:39613 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T09:18:33,469 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46767, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T09:18:33,470 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38901 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7c6d666a4939,39613,1733131113414 2024-12-02T09:18:33,470 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38901 {}] master.ServerManager(517): Registering regionserver=7c6d666a4939,39613,1733131113414 2024-12-02T09:18:33,472 DEBUG [RS:1;7c6d666a4939:39613 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c 2024-12-02T09:18:33,472 DEBUG [RS:1;7c6d666a4939:39613 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40751 2024-12-02T09:18:33,472 DEBUG [RS:1;7c6d666a4939:39613 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T09:18:33,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T09:18:33,478 DEBUG [RS:1;7c6d666a4939:39613 {}] zookeeper.ZKUtil(111): regionserver:39613-0x1009a47ed610002, quorum=127.0.0.1:51860, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7c6d666a4939,39613,1733131113414 2024-12-02T09:18:33,478 WARN [RS:1;7c6d666a4939:39613 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T09:18:33,478 INFO [RS:1;7c6d666a4939:39613 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-02T09:18:33,478 DEBUG [RS:1;7c6d666a4939:39613 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414 2024-12-02T09:18:33,478 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7c6d666a4939,39613,1733131113414] 2024-12-02T09:18:33,482 INFO [RS:1;7c6d666a4939:39613 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T09:18:33,484 INFO [RS:1;7c6d666a4939:39613 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T09:18:33,484 INFO [RS:1;7c6d666a4939:39613 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T09:18:33,485 INFO [RS:1;7c6d666a4939:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:33,485 INFO [RS:1;7c6d666a4939:39613 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T09:18:33,486 INFO [RS:1;7c6d666a4939:39613 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T09:18:33,486 INFO [RS:1;7c6d666a4939:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:33,486 DEBUG [RS:1;7c6d666a4939:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:33,486 DEBUG [RS:1;7c6d666a4939:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:33,486 DEBUG [RS:1;7c6d666a4939:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:33,486 DEBUG [RS:1;7c6d666a4939:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:33,486 DEBUG [RS:1;7c6d666a4939:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:33,486 DEBUG [RS:1;7c6d666a4939:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7c6d666a4939:0, corePoolSize=2, maxPoolSize=2 2024-12-02T09:18:33,486 DEBUG [RS:1;7c6d666a4939:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:33,487 DEBUG [RS:1;7c6d666a4939:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:33,487 DEBUG [RS:1;7c6d666a4939:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:33,487 DEBUG [RS:1;7c6d666a4939:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:33,487 DEBUG [RS:1;7c6d666a4939:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:33,487 DEBUG [RS:1;7c6d666a4939:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:18:33,487 DEBUG [RS:1;7c6d666a4939:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7c6d666a4939:0, corePoolSize=3, maxPoolSize=3 2024-12-02T09:18:33,487 DEBUG [RS:1;7c6d666a4939:39613 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0, corePoolSize=3, maxPoolSize=3 2024-12-02T09:18:33,488 INFO [RS:1;7c6d666a4939:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:33,488 INFO [RS:1;7c6d666a4939:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:33,489 INFO [RS:1;7c6d666a4939:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:33,489 INFO [RS:1;7c6d666a4939:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:33,489 INFO [RS:1;7c6d666a4939:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:33,489 INFO [RS:1;7c6d666a4939:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,39613,1733131113414-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T09:18:33,510 INFO [RS:1;7c6d666a4939:39613 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T09:18:33,510 INFO [RS:1;7c6d666a4939:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,39613,1733131113414-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:33,510 INFO [RS:1;7c6d666a4939:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:33,511 INFO [RS:1;7c6d666a4939:39613 {}] regionserver.Replication(171): 7c6d666a4939,39613,1733131113414 started 2024-12-02T09:18:33,523 INFO [RS:1;7c6d666a4939:39613 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:18:33,523 INFO [RS:1;7c6d666a4939:39613 {}] regionserver.HRegionServer(1482): Serving as 7c6d666a4939,39613,1733131113414, RpcServer on 7c6d666a4939/172.17.0.3:39613, sessionid=0x1009a47ed610002 2024-12-02T09:18:33,524 DEBUG [RS:1;7c6d666a4939:39613 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T09:18:33,524 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;7c6d666a4939:39613,5,FailOnTimeoutGroup] 2024-12-02T09:18:33,524 DEBUG [RS:1;7c6d666a4939:39613 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7c6d666a4939,39613,1733131113414 2024-12-02T09:18:33,524 DEBUG [RS:1;7c6d666a4939:39613 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7c6d666a4939,39613,1733131113414' 2024-12-02T09:18:33,524 DEBUG [RS:1;7c6d666a4939:39613 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T09:18:33,524 INFO [Time-limited test {}] wal.TestLogRolling(207): Replication=2 2024-12-02T09:18:33,524 DEBUG [RS:1;7c6d666a4939:39613 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T09:18:33,525 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T09:18:33,525 DEBUG [RS:1;7c6d666a4939:39613 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T09:18:33,525 DEBUG [RS:1;7c6d666a4939:39613 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T09:18:33,525 DEBUG [RS:1;7c6d666a4939:39613 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7c6d666a4939,39613,1733131113414 2024-12-02T09:18:33,525 DEBUG [RS:1;7c6d666a4939:39613 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7c6d666a4939,39613,1733131113414' 2024-12-02T09:18:33,525 DEBUG [RS:1;7c6d666a4939:39613 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T09:18:33,525 DEBUG [RS:1;7c6d666a4939:39613 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T09:18:33,526 DEBUG [RS:1;7c6d666a4939:39613 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T09:18:33,526 INFO [RS:1;7c6d666a4939:39613 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T09:18:33,526 INFO [RS:1;7c6d666a4939:39613 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T09:18:33,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is 7c6d666a4939,38901,1733131112000 2024-12-02T09:18:33,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7b2ba6f6 2024-12-02T09:18:33,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T09:18:33,528 INFO [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T09:18:33,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38901 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-02T09:18:33,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38901 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-02T09:18:33,529 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38901 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T09:18:33,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38901 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath 2024-12-02T09:18:33,533 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T09:18:33,533 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:33,533 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38901 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRollOnDatanodeDeath" procId is: 4 2024-12-02T09:18:33,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T09:18:33,535 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T09:18:33,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39375 is added to blk_1073741835_1011 (size=393) 2024-12-02T09:18:33,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40489 is added to blk_1073741835_1011 (size=393) 2024-12-02T09:18:33,547 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ed3ba9f85e4101be92817d08be54655d, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c 2024-12-02T09:18:33,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40489 is added to blk_1073741836_1012 (size=76) 2024-12-02T09:18:33,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39375 is added to blk_1073741836_1012 (size=76) 2024-12-02T09:18:33,553 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:18:33,553 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing ed3ba9f85e4101be92817d08be54655d, disabling compactions & flushes 2024-12-02T09:18:33,553 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d. 2024-12-02T09:18:33,553 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d. 2024-12-02T09:18:33,553 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d. after waiting 0 ms 2024-12-02T09:18:33,553 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d. 2024-12-02T09:18:33,553 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d. 2024-12-02T09:18:33,553 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for ed3ba9f85e4101be92817d08be54655d: Waiting for close lock at 1733131113553Disabling compacts and flushes for region at 1733131113553Disabling writes for close at 1733131113553Writing region close event to WAL at 1733131113553Closed at 1733131113553 2024-12-02T09:18:33,555 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T09:18:33,555 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1733131113555"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733131113555"}]},"ts":"1733131113555"} 2024-12-02T09:18:33,557 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T09:18:33,558 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T09:18:33,559 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733131113558"}]},"ts":"1733131113558"} 2024-12-02T09:18:33,561 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta 2024-12-02T09:18:33,561 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=ed3ba9f85e4101be92817d08be54655d, ASSIGN}] 2024-12-02T09:18:33,562 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=ed3ba9f85e4101be92817d08be54655d, ASSIGN 2024-12-02T09:18:33,564 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=ed3ba9f85e4101be92817d08be54655d, ASSIGN; state=OFFLINE, location=7c6d666a4939,37839,1733131112142; forceNewPlan=false, retain=false 2024-12-02T09:18:33,630 INFO [RS:1;7c6d666a4939:39613 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c6d666a4939%2C39613%2C1733131113414, suffix=, logDir=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414, archiveDir=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/oldWALs, maxLogs=32 2024-12-02T09:18:33,633 INFO [RS:1;7c6d666a4939:39613 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C39613%2C1733131113414.1733131113632 2024-12-02T09:18:33,642 INFO [RS:1;7c6d666a4939:39613 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 2024-12-02T09:18:33,646 DEBUG [RS:1;7c6d666a4939:39613 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45099:45099),(127.0.0.1/127.0.0.1:46879:46879)] 2024-12-02T09:18:33,715 INFO [7c6d666a4939:38901 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-02T09:18:33,715 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ed3ba9f85e4101be92817d08be54655d, regionState=OPENING, regionLocation=7c6d666a4939,37839,1733131112142 2024-12-02T09:18:33,719 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=ed3ba9f85e4101be92817d08be54655d, ASSIGN because future has completed 2024-12-02T09:18:33,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed3ba9f85e4101be92817d08be54655d, server=7c6d666a4939,37839,1733131112142}] 2024-12-02T09:18:33,880 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d. 2024-12-02T09:18:33,881 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ed3ba9f85e4101be92817d08be54655d, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d.', STARTKEY => '', ENDKEY => ''} 2024-12-02T09:18:33,881 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath ed3ba9f85e4101be92817d08be54655d 2024-12-02T09:18:33,882 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:18:33,882 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ed3ba9f85e4101be92817d08be54655d 2024-12-02T09:18:33,882 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ed3ba9f85e4101be92817d08be54655d 2024-12-02T09:18:33,883 INFO [StoreOpener-ed3ba9f85e4101be92817d08be54655d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region ed3ba9f85e4101be92817d08be54655d 2024-12-02T09:18:33,885 INFO [StoreOpener-ed3ba9f85e4101be92817d08be54655d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed3ba9f85e4101be92817d08be54655d columnFamilyName info 2024-12-02T09:18:33,885 DEBUG [StoreOpener-ed3ba9f85e4101be92817d08be54655d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:18:33,886 INFO [StoreOpener-ed3ba9f85e4101be92817d08be54655d-1 {}] regionserver.HStore(327): Store=ed3ba9f85e4101be92817d08be54655d/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:18:33,886 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ed3ba9f85e4101be92817d08be54655d 2024-12-02T09:18:33,887 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d 2024-12-02T09:18:33,887 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d 2024-12-02T09:18:33,888 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ed3ba9f85e4101be92817d08be54655d 2024-12-02T09:18:33,888 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ed3ba9f85e4101be92817d08be54655d 2024-12-02T09:18:33,889 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ed3ba9f85e4101be92817d08be54655d 2024-12-02T09:18:33,891 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T09:18:33,892 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ed3ba9f85e4101be92817d08be54655d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=696101, jitterRate=-0.11486200988292694}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T09:18:33,892 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ed3ba9f85e4101be92817d08be54655d 2024-12-02T09:18:33,892 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ed3ba9f85e4101be92817d08be54655d: Running coprocessor pre-open hook at 1733131113882Writing region info on filesystem at 1733131113882Initializing all the Stores at 1733131113883 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131113883Cleaning up temporary data from old regions at 1733131113888 (+5 ms)Running coprocessor post-open hooks at 1733131113892 (+4 ms)Region opened successfully at 1733131113892 2024-12-02T09:18:33,893 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d., pid=6, masterSystemTime=1733131113875 2024-12-02T09:18:33,895 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d. 2024-12-02T09:18:33,896 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d. 2024-12-02T09:18:33,897 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ed3ba9f85e4101be92817d08be54655d, regionState=OPEN, openSeqNum=2, regionLocation=7c6d666a4939,37839,1733131112142 2024-12-02T09:18:33,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed3ba9f85e4101be92817d08be54655d, server=7c6d666a4939,37839,1733131112142 because future has completed 2024-12-02T09:18:33,903 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-02T09:18:33,903 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ed3ba9f85e4101be92817d08be54655d, server=7c6d666a4939,37839,1733131112142 in 180 msec 2024-12-02T09:18:33,906 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-02T09:18:33,906 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=ed3ba9f85e4101be92817d08be54655d, ASSIGN in 342 msec 2024-12-02T09:18:33,907 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T09:18:33,907 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733131113907"}]},"ts":"1733131113907"} 2024-12-02T09:18:33,909 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta 2024-12-02T09:18:33,910 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T09:18:33,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 381 msec 2024-12-02T09:18:38,701 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-02T09:18:38,703 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:18:38,718 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:18:38,721 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:18:38,722 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:18:38,733 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath' 2024-12-02T09:18:39,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-02T09:18:39,834 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-02T09:18:39,835 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath 2024-12-02T09:18:39,835 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer 2024-12-02T09:18:39,835 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T09:18:39,835 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-02T09:18:43,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T09:18:43,613 INFO [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed 2024-12-02T09:18:43,613 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100 2024-12-02T09:18:43,620 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath 2024-12-02T09:18:43,620 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d. 2024-12-02T09:18:43,635 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:18:43,639 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:18:43,640 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:18:43,640 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:18:43,640 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T09:18:43,640 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a9d9bc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:18:43,641 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3abd4c2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:18:43,737 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5e1eaefc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/java.io.tmpdir/jetty-localhost-38189-hadoop-hdfs-3_4_1-tests_jar-_-any-10541424148555734456/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:18:43,738 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@64b89ed5{HTTP/1.1, (http/1.1)}{localhost:38189} 2024-12-02T09:18:43,738 INFO [Time-limited test {}] server.Server(415): Started @120218ms 2024-12-02T09:18:43,739 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T09:18:43,770 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:18:43,774 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:18:43,775 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:18:43,775 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:18:43,775 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T09:18:43,775 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@574823ec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:18:43,776 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@463ced82{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:18:43,878 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4b9d70b4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/java.io.tmpdir/jetty-localhost-39127-hadoop-hdfs-3_4_1-tests_jar-_-any-13506466492750604159/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:18:43,878 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@47432b7b{HTTP/1.1, (http/1.1)}{localhost:39127} 2024-12-02T09:18:43,878 INFO [Time-limited test {}] server.Server(415): Started @120359ms 2024-12-02T09:18:43,879 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T09:18:43,911 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:18:43,913 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:18:43,914 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:18:43,914 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:18:43,914 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T09:18:43,914 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5af75f98{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:18:43,915 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28eb02f0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:18:44,007 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@360084d5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/java.io.tmpdir/jetty-localhost-44111-hadoop-hdfs-3_4_1-tests_jar-_-any-10022242524825061248/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:18:44,007 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@32cdb76b{HTTP/1.1, (http/1.1)}{localhost:44111} 2024-12-02T09:18:44,007 INFO [Time-limited test {}] server.Server(415): Started @120488ms 2024-12-02T09:18:44,008 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T09:18:44,817 WARN [Thread-869 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data5/current/BP-931683727-172.17.0.3-1733131110360/current, will proceed with Du for space computation calculation, 2024-12-02T09:18:44,817 WARN [Thread-870 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data6/current/BP-931683727-172.17.0.3-1733131110360/current, will proceed with Du for space computation calculation, 2024-12-02T09:18:44,833 WARN [Thread-810 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T09:18:44,836 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xab9397290f04c94d with lease ID 0xe9c6559de3b4ceaa: Processing first storage report for DS-45673769-6335-4172-b966-5b5eefbef64f from datanode DatanodeRegistration(127.0.0.1:36861, datanodeUuid=35211fc1-d25b-450d-9a57-97363aeea54f, infoPort=39161, infoSecurePort=0, ipcPort=34117, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360) 2024-12-02T09:18:44,836 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xab9397290f04c94d with lease ID 0xe9c6559de3b4ceaa: from storage DS-45673769-6335-4172-b966-5b5eefbef64f node DatanodeRegistration(127.0.0.1:36861, datanodeUuid=35211fc1-d25b-450d-9a57-97363aeea54f, infoPort=39161, infoSecurePort=0, ipcPort=34117, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:18:44,836 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xab9397290f04c94d with lease ID 0xe9c6559de3b4ceaa: Processing first storage report for DS-cd8b6b4c-5f58-451a-b241-58c22673695f from datanode DatanodeRegistration(127.0.0.1:36861, datanodeUuid=35211fc1-d25b-450d-9a57-97363aeea54f, infoPort=39161, infoSecurePort=0, ipcPort=34117, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360) 2024-12-02T09:18:44,836 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xab9397290f04c94d with lease ID 0xe9c6559de3b4ceaa: from storage DS-cd8b6b4c-5f58-451a-b241-58c22673695f node DatanodeRegistration(127.0.0.1:36861, datanodeUuid=35211fc1-d25b-450d-9a57-97363aeea54f, infoPort=39161, infoSecurePort=0, ipcPort=34117, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-02T09:18:44,940 WARN [Thread-880 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data7/current/BP-931683727-172.17.0.3-1733131110360/current, will proceed with Du for space computation calculation, 2024-12-02T09:18:44,940 WARN [Thread-881 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data8/current/BP-931683727-172.17.0.3-1733131110360/current, will proceed with Du for space computation calculation, 2024-12-02T09:18:44,959 WARN [Thread-832 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T09:18:44,961 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2f19910b688cd08 with lease ID 0xe9c6559de3b4ceab: Processing first storage report for DS-237045ba-35fe-4226-80c0-06528c0e18c0 from datanode DatanodeRegistration(127.0.0.1:45269, datanodeUuid=4b2ed292-06d4-4cfe-a566-0300dc3efcc6, infoPort=33443, infoSecurePort=0, ipcPort=43473, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360) 2024-12-02T09:18:44,961 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2f19910b688cd08 with lease ID 0xe9c6559de3b4ceab: from storage DS-237045ba-35fe-4226-80c0-06528c0e18c0 node DatanodeRegistration(127.0.0.1:45269, datanodeUuid=4b2ed292-06d4-4cfe-a566-0300dc3efcc6, infoPort=33443, infoSecurePort=0, ipcPort=43473, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:18:44,961 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2f19910b688cd08 with lease ID 0xe9c6559de3b4ceab: Processing first storage report for DS-6141eb8f-3aab-430c-bf66-4649f5d81506 from datanode DatanodeRegistration(127.0.0.1:45269, datanodeUuid=4b2ed292-06d4-4cfe-a566-0300dc3efcc6, infoPort=33443, infoSecurePort=0, ipcPort=43473, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360) 2024-12-02T09:18:44,961 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2f19910b688cd08 with lease ID 0xe9c6559de3b4ceab: from storage DS-6141eb8f-3aab-430c-bf66-4649f5d81506 node DatanodeRegistration(127.0.0.1:45269, datanodeUuid=4b2ed292-06d4-4cfe-a566-0300dc3efcc6, infoPort=33443, infoSecurePort=0, ipcPort=43473, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:18:44,991 WARN [Thread-891 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data9/current/BP-931683727-172.17.0.3-1733131110360/current, will proceed with Du for space computation calculation, 2024-12-02T09:18:44,991 WARN [Thread-892 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data10/current/BP-931683727-172.17.0.3-1733131110360/current, will proceed with Du for space computation calculation, 2024-12-02T09:18:45,007 WARN [Thread-854 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T09:18:45,009 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe56211ed9c69114 with lease ID 0xe9c6559de3b4ceac: Processing first storage report for DS-217d50f4-1288-443d-99ce-08122b86cee7 from datanode DatanodeRegistration(127.0.0.1:39031, datanodeUuid=66421c05-42ce-46a5-bee6-581488d30bcb, infoPort=33671, infoSecurePort=0, ipcPort=43689, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360) 2024-12-02T09:18:45,009 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe56211ed9c69114 with lease ID 0xe9c6559de3b4ceac: from storage DS-217d50f4-1288-443d-99ce-08122b86cee7 node DatanodeRegistration(127.0.0.1:39031, datanodeUuid=66421c05-42ce-46a5-bee6-581488d30bcb, infoPort=33671, infoSecurePort=0, ipcPort=43689, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:18:45,010 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe56211ed9c69114 with lease ID 0xe9c6559de3b4ceac: Processing first storage report for DS-f9bf6078-07df-4101-8a1b-b29b11ef2ab7 from datanode DatanodeRegistration(127.0.0.1:39031, datanodeUuid=66421c05-42ce-46a5-bee6-581488d30bcb, infoPort=33671, infoSecurePort=0, ipcPort=43689, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360) 2024-12-02T09:18:45,010 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe56211ed9c69114 with lease ID 0xe9c6559de3b4ceac: from storage DS-f9bf6078-07df-4101-8a1b-b29b11ef2ab7 node DatanodeRegistration(127.0.0.1:39031, datanodeUuid=66421c05-42ce-46a5-bee6-581488d30bcb, infoPort=33671, infoSecurePort=0, ipcPort=43689, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:18:45,036 WARN [ResponseProcessor for block BP-931683727-172.17.0.3-1733131110360:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-931683727-172.17.0.3-1733131110360:blk_1073741837_1013 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:45,036 WARN [ResponseProcessor for block BP-931683727-172.17.0.3-1733131110360:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-931683727-172.17.0.3-1733131110360:blk_1073741830_1006 java.io.IOException: Bad response ERROR for BP-931683727-172.17.0.3-1733131110360:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:45,036 WARN [ResponseProcessor for block BP-931683727-172.17.0.3-1733131110360:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-931683727-172.17.0.3-1733131110360:blk_1073741834_1010 java.io.IOException: Bad response ERROR for BP-931683727-172.17.0.3-1733131110360:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:45,037 WARN [ResponseProcessor for block BP-931683727-172.17.0.3-1733131110360:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-931683727-172.17.0.3-1733131110360:blk_1073741833_1009 java.io.IOException: Bad response ERROR for BP-931683727-172.17.0.3-1733131110360:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:45,037 WARN [DataStreamer for file /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/WALs/7c6d666a4939,38901,1733131112000/7c6d666a4939%2C38901%2C1733131112000.1733131112309 block BP-931683727-172.17.0.3-1733131110360:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK], DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK]) is bad. 2024-12-02T09:18:45,037 WARN [DataStreamer for file /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta block BP-931683727-172.17.0.3-1733131110360:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK], DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK]) is bad. 2024-12-02T09:18:45,037 WARN [DataStreamer for file /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131112776 block BP-931683727-172.17.0.3-1733131110360:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK], DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK]) is bad. 2024-12-02T09:18:45,038 WARN [DataStreamer for file /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 block BP-931683727-172.17.0.3-1733131110360:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK], DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK]) is bad. 2024-12-02T09:18:45,037 WARN [PacketResponder: BP-931683727-172.17.0.3-1733131110360:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:39375] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:45,037 WARN [PacketResponder: BP-931683727-172.17.0.3-1733131110360:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:39375] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Broken pipe at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:45,039 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_363540338_22 at /127.0.0.1:41228 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:40489:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41228 dst: /127.0.0.1:40489 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:45,039 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1691682189_22 at /127.0.0.1:48918 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:39375:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48918 dst: /127.0.0.1:39375 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:45,040 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:41284 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:40489:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41284 dst: /127.0.0.1:40489 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:45,040 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1691682189_22 at /127.0.0.1:41302 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:40489:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41302 dst: /127.0.0.1:40489 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:45,039 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:41256 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:40489:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41256 dst: /127.0.0.1:40489 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:45,041 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@544fa662{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:18:45,041 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:48880 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:39375:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48880 dst: /127.0.0.1:39375 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:45,042 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3574ce3f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:18:45,042 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:18:45,043 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@10a4d310{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:18:45,040 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:48878 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:39375:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48878 dst: /127.0.0.1:39375 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:45,043 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3d5daa57{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/hadoop.log.dir/,STOPPED} 2024-12-02T09:18:45,043 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_363540338_22 at /127.0.0.1:48862 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:39375:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48862 dst: /127.0.0.1:39375 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:45,045 WARN [BP-931683727-172.17.0.3-1733131110360 heartbeating to localhost/127.0.0.1:40751 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T09:18:45,045 WARN [BP-931683727-172.17.0.3-1733131110360 heartbeating to localhost/127.0.0.1:40751 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-931683727-172.17.0.3-1733131110360 (Datanode Uuid 900a00a2-9ff5-4c37-9912-5024947f217c) service to localhost/127.0.0.1:40751 2024-12-02T09:18:45,046 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T09:18:45,046 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data3/current/BP-931683727-172.17.0.3-1733131110360 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:18:45,046 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T09:18:45,046 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data4/current/BP-931683727-172.17.0.3-1733131110360 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:18:45,047 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T09:18:45,052 WARN [DataStreamer for file /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131112776 block BP-931683727-172.17.0.3-1733131110360:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009 java.net.SocketException: Connection reset at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?] at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?] at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:45,052 WARN [DataStreamer for file /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/WALs/7c6d666a4939,38901,1733131112000/7c6d666a4939%2C38901%2C1733131112000.1733131112309 block BP-931683727-172.17.0.3-1733131110360:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006 java.net.SocketException: Connection reset at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?] at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?] at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:45,052 WARN [DataStreamer for file /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta block BP-931683727-172.17.0.3-1733131110360:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010 java.net.SocketException: Connection reset at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?] at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?] at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:45,054 ERROR [org.apache.hadoop.hdfs.server.datanode.DataXceiver@23c6d266 {}] datanode.DataXceiver(331): 127.0.0.1:40489:DataXceiver error processing unknown operation src: /127.0.0.1:39198 dst: /127.0.0.1:40489 java.io.IOException: Server closed. at org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.addPeer(DataXceiverServer.java:334) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:232) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:45,054 WARN [DataStreamer for file /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 block BP-931683727-172.17.0.3-1733131110360:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013 java.net.SocketException: Connection reset at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?] at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?] at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:45,056 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3e10767c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:18:45,056 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4d2d9832{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:18:45,056 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:18:45,056 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@48d478e7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:18:45,056 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@651aa118{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/hadoop.log.dir/,STOPPED} 2024-12-02T09:18:45,058 WARN [BP-931683727-172.17.0.3-1733131110360 heartbeating to localhost/127.0.0.1:40751 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T09:18:45,058 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T09:18:45,058 WARN [BP-931683727-172.17.0.3-1733131110360 heartbeating to localhost/127.0.0.1:40751 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-931683727-172.17.0.3-1733131110360 (Datanode Uuid 4f2da047-ed4a-4df8-9d20-04093f4eaca4) service to localhost/127.0.0.1:40751 2024-12-02T09:18:45,058 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T09:18:45,059 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data1/current/BP-931683727-172.17.0.3-1733131110360 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:18:45,059 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data2/current/BP-931683727-172.17.0.3-1733131110360 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:18:45,059 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T09:18:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d., hostname=7c6d666a4939,37839,1733131112142, seqNum=2] 2024-12-02T09:18:45,065 ERROR [FSHLog-0-hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c-prefix:7c6d666a4939,37839,1733131112142 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:45,065 WARN [FSHLog-0-hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c-prefix:7c6d666a4939,37839,1733131112142 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:45,065 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 7c6d666a4939%2C37839%2C1733131112142:(num 1733131112776) roll requested 2024-12-02T09:18:45,066 INFO [regionserver/7c6d666a4939:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C37839%2C1733131112142.1733131125066 2024-12-02T09:18:45,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:45,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:45,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:45,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:45,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:45,075 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131112776 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131125066 2024-12-02T09:18:45,075 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:45,076 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:45,077 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33443:33443),(127.0.0.1/127.0.0.1:33671:33671)] 2024-12-02T09:18:45,077 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131112776 is not closed yet, will try archiving it next time 2024-12-02T09:18:45,077 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-02T09:18:45,077 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-02T09:18:45,077 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131112776 2024-12-02T09:18:45,081 WARN [IPC Server handler 2 on default port 40751 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131112776 has not been closed. Lease recovery is in progress. RecoveryId = 1019 for block blk_1073741833_1009 2024-12-02T09:18:45,085 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131112776 after 4ms 2024-12-02T09:18:45,488 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:46,285 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:47,077 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:47,085 INFO [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131125066 2024-12-02T09:18:47,086 WARN [ResponseProcessor for block BP-931683727-172.17.0.3-1733131110360:blk_1073741838_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-931683727-172.17.0.3-1733131110360:blk_1073741838_1018 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:47,086 WARN [DataStreamer for file /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131125066 block BP-931683727-172.17.0.3-1733131110360:blk_1073741838_1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK], DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK]) is bad. 2024-12-02T09:18:47,087 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:58914 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:45269:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58914 dst: /127.0.0.1:45269 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:47,087 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:37926 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:39031:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37926 dst: /127.0.0.1:39031 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:47,103 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4b9d70b4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:18:47,104 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@47432b7b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:18:47,104 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:18:47,104 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@463ced82{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:18:47,104 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@574823ec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/hadoop.log.dir/,STOPPED} 2024-12-02T09:18:47,105 WARN [BP-931683727-172.17.0.3-1733131110360 heartbeating to localhost/127.0.0.1:40751 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T09:18:47,105 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T09:18:47,105 WARN [BP-931683727-172.17.0.3-1733131110360 heartbeating to localhost/127.0.0.1:40751 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-931683727-172.17.0.3-1733131110360 (Datanode Uuid 4b2ed292-06d4-4cfe-a566-0300dc3efcc6) service to localhost/127.0.0.1:40751 2024-12-02T09:18:47,105 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T09:18:47,106 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data7/current/BP-931683727-172.17.0.3-1733131110360 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:18:47,106 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data8/current/BP-931683727-172.17.0.3-1733131110360 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:18:47,106 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T09:18:47,489 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:48,286 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:49,078 WARN [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK]] 2024-12-02T09:18:49,078 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:49,079 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 7c6d666a4939%2C37839%2C1733131112142:(num 1733131125066) roll requested 2024-12-02T09:18:49,079 INFO [regionserver/7c6d666a4939:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C37839%2C1733131112142.1733131129079 2024-12-02T09:18:49,082 WARN [Thread-912 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741839_1021 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:49,082 WARN [Thread-912 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK], DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK]) is bad. 2024-12-02T09:18:49,082 WARN [Thread-912 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741839_1021 2024-12-02T09:18:49,085 WARN [Thread-912 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK] 2024-12-02T09:18:49,085 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131112776 after 4007ms 2024-12-02T09:18:49,090 WARN [Thread-912 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741840_1022 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40489 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:49,090 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:49962 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741840_1022] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data6]'}, localName='127.0.0.1:36861', datanodeUuid='35211fc1-d25b-450d-9a57-97363aeea54f', xmitsInProgress=0}:Exception transferring block BP-931683727-172.17.0.3-1733131110360:blk_1073741840_1022 to mirror 127.0.0.1:40489 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:49,091 WARN [Thread-912 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741840_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK], DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]) is bad. 2024-12-02T09:18:49,091 WARN [Thread-912 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741840_1022 2024-12-02T09:18:49,091 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:49962 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741840_1022] {}] datanode.BlockReceiver(316): Block 1073741840 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-02T09:18:49,091 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:49962 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741840_1022] {}] datanode.DataXceiver(331): 127.0.0.1:36861:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49962 dst: /127.0.0.1:36861 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:49,091 WARN [Thread-912 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK] 2024-12-02T09:18:49,093 WARN [Thread-912 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1023 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:49,093 WARN [Thread-912 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741841_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK], DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK]) is bad. 2024-12-02T09:18:49,093 WARN [Thread-912 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741841_1023 2024-12-02T09:18:49,094 WARN [Thread-912 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK] 2024-12-02T09:18:49,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:49,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:49,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:49,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:49,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:49,098 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131125066 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131129079 2024-12-02T09:18:49,099 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39161:39161),(127.0.0.1/127.0.0.1:33671:33671)] 2024-12-02T09:18:49,099 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131112776 is not closed yet, will try archiving it next time 2024-12-02T09:18:49,099 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131125066 is not closed yet, will try archiving it next time 2024-12-02T09:18:49,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39031 is added to blk_1073741838_1020 (size=2431) 2024-12-02T09:18:49,112 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-12-02T09:18:49,490 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:49,501 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131112776 is not closed yet, will try archiving it next time 2024-12-02T09:18:50,287 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:51,100 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:51,117 WARN [ResponseProcessor for block BP-931683727-172.17.0.3-1733131110360:blk_1073741842_1024 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-931683727-172.17.0.3-1733131110360:blk_1073741842_1024 java.io.IOException: Bad response ERROR for BP-931683727-172.17.0.3-1733131110360:blk_1073741842_1024 from datanode DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:51,118 WARN [DataStreamer for file /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131129079 block BP-931683727-172.17.0.3-1733131110360:blk_1073741842_1024 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741842_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK], DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK]) is bad. 2024-12-02T09:18:51,118 WARN [PacketResponder: BP-931683727-172.17.0.3-1733131110360:blk_1073741842_1024, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:39031] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:51,119 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:49964 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741842_1024] {}] datanode.DataXceiver(331): 127.0.0.1:36861:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49964 dst: /127.0.0.1:36861 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:51,119 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:39754 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741842_1024] {}] datanode.DataXceiver(331): 127.0.0.1:39031:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39754 dst: /127.0.0.1:39031 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:51,120 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@360084d5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:18:51,121 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@32cdb76b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:18:51,121 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:18:51,121 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28eb02f0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:18:51,121 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5af75f98{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/hadoop.log.dir/,STOPPED} 2024-12-02T09:18:51,123 WARN [BP-931683727-172.17.0.3-1733131110360 heartbeating to localhost/127.0.0.1:40751 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T09:18:51,124 WARN [BP-931683727-172.17.0.3-1733131110360 heartbeating to localhost/127.0.0.1:40751 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-931683727-172.17.0.3-1733131110360 (Datanode Uuid 66421c05-42ce-46a5-bee6-581488d30bcb) service to localhost/127.0.0.1:40751 2024-12-02T09:18:51,124 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T09:18:51,124 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T09:18:51,125 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data9/current/BP-931683727-172.17.0.3-1733131110360 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:18:51,125 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data10/current/BP-931683727-172.17.0.3-1733131110360 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:18:51,125 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T09:18:51,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37839 {}] regionserver.HRegion(8855): Flush requested on ed3ba9f85e4101be92817d08be54655d 2024-12-02T09:18:51,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed3ba9f85e4101be92817d08be54655d 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-02T09:18:51,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/.tmp/info/8d0a8c7afa3044419e89d6ab42aa0399 is 1080, key is row0002/info:/1733131127108/Put/seqid=0 2024-12-02T09:18:51,155 WARN [Thread-922 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1026 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:51,155 WARN [Thread-922 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741843_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK], DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK]) is bad. 2024-12-02T09:18:51,155 WARN [Thread-922 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741843_1026 2024-12-02T09:18:51,155 WARN [Thread-922 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK] 2024-12-02T09:18:51,156 WARN [Thread-922 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:51,157 WARN [Thread-922 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK], DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK]) is bad. 2024-12-02T09:18:51,157 WARN [Thread-922 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741844_1027 2024-12-02T09:18:51,157 WARN [Thread-922 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK] 2024-12-02T09:18:51,159 WARN [Thread-922 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:39375 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:51,159 WARN [Thread-922 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK], DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK]) is bad. 2024-12-02T09:18:51,159 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:49970 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741845_1028] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data6]'}, localName='127.0.0.1:36861', datanodeUuid='35211fc1-d25b-450d-9a57-97363aeea54f', xmitsInProgress=0}:Exception transferring block BP-931683727-172.17.0.3-1733131110360:blk_1073741845_1028 to mirror 127.0.0.1:39375 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:51,159 WARN [Thread-922 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741845_1028 2024-12-02T09:18:51,160 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:49970 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741845_1028] {}] datanode.BlockReceiver(316): Block 1073741845 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-02T09:18:51,160 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:49970 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741845_1028] {}] datanode.DataXceiver(331): 127.0.0.1:36861:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49970 dst: /127.0.0.1:36861 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:51,160 WARN [Thread-922 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK] 2024-12-02T09:18:51,162 WARN [Thread-922 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40489 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:51,162 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:49982 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741846_1029] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data6]'}, localName='127.0.0.1:36861', datanodeUuid='35211fc1-d25b-450d-9a57-97363aeea54f', xmitsInProgress=0}:Exception transferring block BP-931683727-172.17.0.3-1733131110360:blk_1073741846_1029 to mirror 127.0.0.1:40489 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:51,163 WARN [Thread-922 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK], DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]) is bad. 2024-12-02T09:18:51,163 WARN [Thread-922 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741846_1029 2024-12-02T09:18:51,163 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:49982 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741846_1029] {}] datanode.BlockReceiver(316): Block 1073741846 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-02T09:18:51,163 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:49982 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741846_1029] {}] datanode.DataXceiver(331): 127.0.0.1:36861:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49982 dst: /127.0.0.1:36861 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:51,163 WARN [Thread-922 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK] 2024-12-02T09:18:51,164 WARN [IPC Server handler 1 on default port 40751 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-02T09:18:51,164 WARN [IPC Server handler 1 on default port 40751 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-02T09:18:51,164 WARN [IPC Server handler 1 on default port 40751 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-02T09:18:51,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741847_1030 (size=10347) 2024-12-02T09:18:51,490 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:51,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/.tmp/info/8d0a8c7afa3044419e89d6ab42aa0399 2024-12-02T09:18:51,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/.tmp/info/8d0a8c7afa3044419e89d6ab42aa0399 as hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/8d0a8c7afa3044419e89d6ab42aa0399 2024-12-02T09:18:51,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/8d0a8c7afa3044419e89d6ab42aa0399, entries=5, sequenceid=11, filesize=10.1 K 2024-12-02T09:18:51,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for ed3ba9f85e4101be92817d08be54655d in 448ms, sequenceid=11, compaction requested=false 2024-12-02T09:18:51,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed3ba9f85e4101be92817d08be54655d: 2024-12-02T09:18:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37839 {}] regionserver.HRegion(8855): Flush requested on ed3ba9f85e4101be92817d08be54655d 2024-12-02T09:18:51,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed3ba9f85e4101be92817d08be54655d 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB 2024-12-02T09:18:51,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/.tmp/info/5b8bd58425ca475abbfda4b9fe58edcd is 1080, key is row0007/info:/1733131131136/Put/seqid=0 2024-12-02T09:18:51,767 WARN [Thread-929 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:51,767 WARN [Thread-929 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK], DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK]) is bad. 2024-12-02T09:18:51,767 WARN [Thread-929 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741848_1031 2024-12-02T09:18:51,768 WARN [Thread-929 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK] 2024-12-02T09:18:51,769 WARN [Thread-929 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:51,769 WARN [Thread-929 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK], DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK]) is bad. 2024-12-02T09:18:51,769 WARN [Thread-929 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741849_1032 2024-12-02T09:18:51,770 WARN [Thread-929 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK] 2024-12-02T09:18:51,772 WARN [Thread-929 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:51,772 WARN [Thread-929 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK], DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK]) is bad. 2024-12-02T09:18:51,772 WARN [Thread-929 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741850_1033 2024-12-02T09:18:51,773 WARN [Thread-929 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK] 2024-12-02T09:18:51,775 WARN [Thread-929 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40489 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:51,775 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:50018 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741851_1034] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data6]'}, localName='127.0.0.1:36861', datanodeUuid='35211fc1-d25b-450d-9a57-97363aeea54f', xmitsInProgress=0}:Exception transferring block BP-931683727-172.17.0.3-1733131110360:blk_1073741851_1034 to mirror 127.0.0.1:40489 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:51,775 WARN [Thread-929 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK], DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]) is bad. 2024-12-02T09:18:51,776 WARN [Thread-929 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741851_1034 2024-12-02T09:18:51,776 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:50018 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741851_1034] {}] datanode.BlockReceiver(316): Block 1073741851 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-02T09:18:51,776 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:50018 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741851_1034] {}] datanode.DataXceiver(331): 127.0.0.1:36861:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50018 dst: /127.0.0.1:36861 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:51,776 WARN [Thread-929 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK] 2024-12-02T09:18:51,777 WARN [IPC Server handler 1 on default port 40751 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-02T09:18:51,777 WARN [IPC Server handler 1 on default port 40751 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-02T09:18:51,777 WARN [IPC Server handler 1 on default port 40751 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-02T09:18:51,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741852_1035 (size=12506) 2024-12-02T09:18:52,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/.tmp/info/5b8bd58425ca475abbfda4b9fe58edcd 2024-12-02T09:18:52,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/.tmp/info/5b8bd58425ca475abbfda4b9fe58edcd as hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/5b8bd58425ca475abbfda4b9fe58edcd 2024-12-02T09:18:52,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/5b8bd58425ca475abbfda4b9fe58edcd, entries=7, sequenceid=24, filesize=12.2 K 2024-12-02T09:18:52,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2150 for ed3ba9f85e4101be92817d08be54655d in 436ms, sequenceid=24, compaction requested=false 2024-12-02T09:18:52,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed3ba9f85e4101be92817d08be54655d: 2024-12-02T09:18:52,196 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K 2024-12-02T09:18:52,196 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-02T09:18:52,196 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/5b8bd58425ca475abbfda4b9fe58edcd because midkey is the same as first or last row 2024-12-02T09:18:52,287 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:52,844 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@6b6e383c[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36861, datanodeUuid=35211fc1-d25b-450d-9a57-97363aeea54f, infoPort=39161, infoSecurePort=0, ipcPort=34117, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360):Failed to transfer BP-931683727-172.17.0.3-1733131110360:blk_1073741852_1035 to 127.0.0.1:39031 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:52,844 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@3efe6249[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36861, datanodeUuid=35211fc1-d25b-450d-9a57-97363aeea54f, infoPort=39161, infoSecurePort=0, ipcPort=34117, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360):Failed to transfer BP-931683727-172.17.0.3-1733131110360:blk_1073741847_1030 to 127.0.0.1:45269 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:53,100 WARN [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK]] 2024-12-02T09:18:53,100 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:53,100 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 7c6d666a4939%2C37839%2C1733131112142:(num 1733131129079) roll requested 2024-12-02T09:18:53,101 INFO [regionserver/7c6d666a4939:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C37839%2C1733131112142.1733131133100 2024-12-02T09:18:53,103 WARN [Thread-938 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:53,104 WARN [Thread-938 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK], DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK]) is bad. 2024-12-02T09:18:53,104 WARN [Thread-938 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741853_1036 2024-12-02T09:18:53,104 WARN [Thread-938 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK] 2024-12-02T09:18:53,105 WARN [Thread-938 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:53,105 WARN [Thread-938 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK], DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK]) is bad. 2024-12-02T09:18:53,105 WARN [Thread-938 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741854_1037 2024-12-02T09:18:53,106 WARN [Thread-938 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK] 2024-12-02T09:18:53,107 WARN [Thread-938 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:53,107 WARN [Thread-938 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK], DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK]) is bad. 2024-12-02T09:18:53,107 WARN [Thread-938 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741855_1038 2024-12-02T09:18:53,107 WARN [Thread-938 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK] 2024-12-02T09:18:53,109 WARN [Thread-938 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40489 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:53,109 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:50042 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741856_1039] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data6]'}, localName='127.0.0.1:36861', datanodeUuid='35211fc1-d25b-450d-9a57-97363aeea54f', xmitsInProgress=0}:Exception transferring block BP-931683727-172.17.0.3-1733131110360:blk_1073741856_1039 to mirror 127.0.0.1:40489 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:53,110 WARN [Thread-938 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK], DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]) is bad. 2024-12-02T09:18:53,110 WARN [Thread-938 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741856_1039 2024-12-02T09:18:53,110 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:50042 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741856_1039] {}] datanode.BlockReceiver(316): Block 1073741856 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-02T09:18:53,110 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:50042 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741856_1039] {}] datanode.DataXceiver(331): 127.0.0.1:36861:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50042 dst: /127.0.0.1:36861 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:53,110 WARN [Thread-938 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK] 2024-12-02T09:18:53,111 WARN [IPC Server handler 0 on default port 40751 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-02T09:18:53,111 WARN [IPC Server handler 0 on default port 40751 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-02T09:18:53,111 WARN [IPC Server handler 0 on default port 40751 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-02T09:18:53,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:53,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:53,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:53,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:53,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:53,114 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131129079 with entries=25, filesize=25.38 KB; new WAL /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131133100 2024-12-02T09:18:53,115 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39161:39161)] 2024-12-02T09:18:53,115 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131112776 is not closed yet, will try archiving it next time 2024-12-02T09:18:53,115 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131129079 is not closed yet, will try archiving it next time 2024-12-02T09:18:53,115 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131125066 to hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/oldWALs/7c6d666a4939%2C37839%2C1733131112142.1733131125066 2024-12-02T09:18:53,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741842_1025 (size=25992) 2024-12-02T09:18:53,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37839 {}] regionserver.HRegion(8855): Flush requested on ed3ba9f85e4101be92817d08be54655d 2024-12-02T09:18:53,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed3ba9f85e4101be92817d08be54655d 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-12-02T09:18:53,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/.tmp/info/0831543502bb4968927c17a3d1b41baf is 1079, key is tmprow/info:/1733131133178/Put/seqid=0 2024-12-02T09:18:53,186 WARN [Thread-943 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:53,186 WARN [Thread-943 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK], DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK]) is bad. 2024-12-02T09:18:53,186 WARN [Thread-943 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741858_1041 2024-12-02T09:18:53,187 WARN [Thread-943 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK] 2024-12-02T09:18:53,188 WARN [Thread-943 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:53,188 WARN [Thread-943 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK], DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]) is bad. 2024-12-02T09:18:53,188 WARN [Thread-943 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741859_1042 2024-12-02T09:18:53,188 WARN [Thread-943 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK] 2024-12-02T09:18:53,189 WARN [Thread-943 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:53,190 WARN [Thread-943 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK], DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK]) is bad. 2024-12-02T09:18:53,190 WARN [Thread-943 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741860_1043 2024-12-02T09:18:53,190 WARN [Thread-943 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK] 2024-12-02T09:18:53,192 WARN [Thread-943 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:53,192 WARN [Thread-943 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK], DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK]) is bad. 2024-12-02T09:18:53,192 WARN [Thread-943 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741861_1044 2024-12-02T09:18:53,192 WARN [Thread-943 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK] 2024-12-02T09:18:53,193 WARN [IPC Server handler 3 on default port 40751 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-02T09:18:53,193 WARN [IPC Server handler 3 on default port 40751 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-02T09:18:53,193 WARN [IPC Server handler 3 on default port 40751 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-02T09:18:53,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741862_1045 (size=6027) 2024-12-02T09:18:53,491 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:53,517 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131112776 is not closed yet, will try archiving it next time 2024-12-02T09:18:53,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/.tmp/info/0831543502bb4968927c17a3d1b41baf 2024-12-02T09:18:53,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/.tmp/info/0831543502bb4968927c17a3d1b41baf as hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/0831543502bb4968927c17a3d1b41baf 2024-12-02T09:18:53,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/0831543502bb4968927c17a3d1b41baf, entries=1, sequenceid=34, filesize=5.9 K 2024-12-02T09:18:53,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for ed3ba9f85e4101be92817d08be54655d in 443ms, sequenceid=34, compaction requested=true 2024-12-02T09:18:53,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed3ba9f85e4101be92817d08be54655d: 2024-12-02T09:18:53,623 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K 2024-12-02T09:18:53,623 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-02T09:18:53,623 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/5b8bd58425ca475abbfda4b9fe58edcd because midkey is the same as first or last row 2024-12-02T09:18:53,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed3ba9f85e4101be92817d08be54655d:info, priority=-2147483648, current under compaction store size is 1 2024-12-02T09:18:53,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:18:53,623 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T09:18:53,625 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T09:18:53,625 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.HStore(1541): ed3ba9f85e4101be92817d08be54655d/info is initiating minor compaction (all files) 2024-12-02T09:18:53,625 INFO [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed3ba9f85e4101be92817d08be54655d/info in TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d. 2024-12-02T09:18:53,625 INFO [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/8d0a8c7afa3044419e89d6ab42aa0399, hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/5b8bd58425ca475abbfda4b9fe58edcd, hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/0831543502bb4968927c17a3d1b41baf] into tmpdir=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/.tmp, totalSize=28.2 K 2024-12-02T09:18:53,626 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d0a8c7afa3044419e89d6ab42aa0399, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733131127108 2024-12-02T09:18:53,626 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b8bd58425ca475abbfda4b9fe58edcd, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1733131131136 2024-12-02T09:18:53,627 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0831543502bb4968927c17a3d1b41baf, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733131133178 2024-12-02T09:18:53,640 INFO [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed3ba9f85e4101be92817d08be54655d#info#compaction#21 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T09:18:53,640 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/.tmp/info/e8e3c0ec93a84097ad7a6d97d6dac64c is 1080, key is row0002/info:/1733131127108/Put/seqid=0 2024-12-02T09:18:53,642 WARN [Thread-949 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:53,642 WARN [Thread-949 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK], DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]) is bad. 2024-12-02T09:18:53,642 WARN [Thread-949 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741863_1046 2024-12-02T09:18:53,643 WARN [Thread-949 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK] 2024-12-02T09:18:53,644 WARN [Thread-949 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1047 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:53,644 WARN [Thread-949 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741864_1047 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK], DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK]) is bad. 2024-12-02T09:18:53,644 WARN [Thread-949 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741864_1047 2024-12-02T09:18:53,645 WARN [Thread-949 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK] 2024-12-02T09:18:53,646 WARN [Thread-949 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741865_1048 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:53,646 WARN [Thread-949 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741865_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK], DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK]) is bad. 2024-12-02T09:18:53,646 WARN [Thread-949 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741865_1048 2024-12-02T09:18:53,646 WARN [Thread-949 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK] 2024-12-02T09:18:53,648 WARN [Thread-949 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1049 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:53,648 WARN [Thread-949 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741866_1049 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK], DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK]) is bad. 2024-12-02T09:18:53,648 WARN [Thread-949 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741866_1049 2024-12-02T09:18:53,648 WARN [Thread-949 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK] 2024-12-02T09:18:53,649 WARN [IPC Server handler 1 on default port 40751 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-02T09:18:53,649 WARN [IPC Server handler 1 on default port 40751 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-02T09:18:53,649 WARN [IPC Server handler 1 on default port 40751 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-02T09:18:53,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741867_1050 (size=17994) 2024-12-02T09:18:54,062 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/.tmp/info/e8e3c0ec93a84097ad7a6d97d6dac64c as hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/e8e3c0ec93a84097ad7a6d97d6dac64c 2024-12-02T09:18:54,071 INFO [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed3ba9f85e4101be92817d08be54655d/info of ed3ba9f85e4101be92817d08be54655d into e8e3c0ec93a84097ad7a6d97d6dac64c(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T09:18:54,071 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed3ba9f85e4101be92817d08be54655d: 2024-12-02T09:18:54,071 INFO [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d., storeName=ed3ba9f85e4101be92817d08be54655d/info, priority=13, startTime=1733131133623; duration=0sec 2024-12-02T09:18:54,071 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K 2024-12-02T09:18:54,071 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-02T09:18:54,071 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/e8e3c0ec93a84097ad7a6d97d6dac64c because midkey is the same as first or last row 2024-12-02T09:18:54,071 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K 2024-12-02T09:18:54,071 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-02T09:18:54,071 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/e8e3c0ec93a84097ad7a6d97d6dac64c because midkey is the same as first or last row 2024-12-02T09:18:54,071 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K 2024-12-02T09:18:54,071 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-02T09:18:54,071 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/e8e3c0ec93a84097ad7a6d97d6dac64c because midkey is the same as first or last row 2024-12-02T09:18:54,071 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:18:54,071 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed3ba9f85e4101be92817d08be54655d:info 2024-12-02T09:18:54,288 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:54,601 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed3ba9f85e4101be92817d08be54655d 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-12-02T09:18:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37839 {}] regionserver.HRegion(8855): Flush requested on ed3ba9f85e4101be92817d08be54655d 2024-12-02T09:18:54,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/.tmp/info/e6e7951f47444803857f432b3a8e048b is 1079, key is tmprow/info:/1733131134600/Put/seqid=0 2024-12-02T09:18:54,608 WARN [Thread-953 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1051 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:54,608 WARN [Thread-953 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741868_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK], DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]) is bad. 2024-12-02T09:18:54,608 WARN [Thread-953 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741868_1051 2024-12-02T09:18:54,609 WARN [Thread-953 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK] 2024-12-02T09:18:54,611 WARN [Thread-953 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1052 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:54,611 WARN [Thread-953 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741869_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK], DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK]) is bad. 2024-12-02T09:18:54,611 WARN [Thread-953 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741869_1052 2024-12-02T09:18:54,612 WARN [Thread-953 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK] 2024-12-02T09:18:54,613 WARN [Thread-953 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741870_1053 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:54,614 WARN [Thread-953 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741870_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK], DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK]) is bad. 2024-12-02T09:18:54,614 WARN [Thread-953 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741870_1053 2024-12-02T09:18:54,614 WARN [Thread-953 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK] 2024-12-02T09:18:54,616 WARN [Thread-953 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741871_1054 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:54,616 WARN [Thread-953 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741871_1054 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK], DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK]) is bad. 2024-12-02T09:18:54,616 WARN [Thread-953 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741871_1054 2024-12-02T09:18:54,617 WARN [Thread-953 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK] 2024-12-02T09:18:54,618 WARN [IPC Server handler 4 on default port 40751 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-02T09:18:54,618 WARN [IPC Server handler 4 on default port 40751 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-02T09:18:54,618 WARN [IPC Server handler 4 on default port 40751 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-02T09:18:54,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741872_1055 (size=6027) 2024-12-02T09:18:54,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/.tmp/info/e6e7951f47444803857f432b3a8e048b 2024-12-02T09:18:54,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/.tmp/info/e6e7951f47444803857f432b3a8e048b as hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/e6e7951f47444803857f432b3a8e048b 2024-12-02T09:18:54,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/e6e7951f47444803857f432b3a8e048b, entries=1, sequenceid=45, filesize=5.9 K 2024-12-02T09:18:54,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=0 B/0 for ed3ba9f85e4101be92817d08be54655d in 49ms, sequenceid=45, compaction requested=false 2024-12-02T09:18:54,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed3ba9f85e4101be92817d08be54655d: 2024-12-02T09:18:54,651 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K 2024-12-02T09:18:54,651 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-02T09:18:54,651 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/e8e3c0ec93a84097ad7a6d97d6dac64c because midkey is the same as first or last row 2024-12-02T09:18:55,115 WARN [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK]] 2024-12-02T09:18:55,115 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:55,116 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 7c6d666a4939%2C37839%2C1733131112142:(num 1733131133100) roll requested 2024-12-02T09:18:55,116 INFO [regionserver/7c6d666a4939:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C37839%2C1733131112142.1733131135116 2024-12-02T09:18:55,119 WARN [Thread-957 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1056 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:55,119 WARN [Thread-957 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741873_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK], DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK]) is bad. 2024-12-02T09:18:55,119 WARN [Thread-957 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741873_1056 2024-12-02T09:18:55,120 WARN [Thread-957 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK] 2024-12-02T09:18:55,123 WARN [Thread-957 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741874_1057 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:39375 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:55,122 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:50108 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741874_1057] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data6]'}, localName='127.0.0.1:36861', datanodeUuid='35211fc1-d25b-450d-9a57-97363aeea54f', xmitsInProgress=0}:Exception transferring block BP-931683727-172.17.0.3-1733131110360:blk_1073741874_1057 to mirror 127.0.0.1:39375 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:55,123 WARN [Thread-957 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741874_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK], DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK]) is bad. 2024-12-02T09:18:55,123 WARN [Thread-957 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741874_1057 2024-12-02T09:18:55,123 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:50108 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741874_1057] {}] datanode.BlockReceiver(316): Block 1073741874 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-02T09:18:55,123 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:50108 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741874_1057] {}] datanode.DataXceiver(331): 127.0.0.1:36861:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50108 dst: /127.0.0.1:36861 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:55,124 WARN [Thread-957 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK] 2024-12-02T09:18:55,126 WARN [Thread-957 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741875_1058 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:45269 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:55,126 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:50110 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741875_1058] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data6]'}, localName='127.0.0.1:36861', datanodeUuid='35211fc1-d25b-450d-9a57-97363aeea54f', xmitsInProgress=0}:Exception transferring block BP-931683727-172.17.0.3-1733131110360:blk_1073741875_1058 to mirror 127.0.0.1:45269 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:55,127 WARN [Thread-957 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741875_1058 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK], DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK]) is bad. 2024-12-02T09:18:55,127 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:50110 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741875_1058] {}] datanode.BlockReceiver(316): Block 1073741875 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-02T09:18:55,127 WARN [Thread-957 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741875_1058 2024-12-02T09:18:55,127 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:50110 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741875_1058] {}] datanode.DataXceiver(331): 127.0.0.1:36861:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50110 dst: /127.0.0.1:36861 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:55,128 WARN [Thread-957 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK] 2024-12-02T09:18:55,130 WARN [Thread-957 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741876_1059 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40489 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:55,130 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:50112 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741876_1059] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data6]'}, localName='127.0.0.1:36861', datanodeUuid='35211fc1-d25b-450d-9a57-97363aeea54f', xmitsInProgress=0}:Exception transferring block BP-931683727-172.17.0.3-1733131110360:blk_1073741876_1059 to mirror 127.0.0.1:40489 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:55,131 WARN [Thread-957 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741876_1059 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK], DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]) is bad. 2024-12-02T09:18:55,131 WARN [Thread-957 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741876_1059 2024-12-02T09:18:55,131 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:50112 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741876_1059] {}] datanode.BlockReceiver(316): Block 1073741876 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-12-02T09:18:55,131 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:50112 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741876_1059] {}] datanode.DataXceiver(331): 127.0.0.1:36861:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50112 dst: /127.0.0.1:36861 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:55,131 WARN [Thread-957 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK] 2024-12-02T09:18:55,132 WARN [IPC Server handler 0 on default port 40751 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-02T09:18:55,132 WARN [IPC Server handler 0 on default port 40751 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-02T09:18:55,132 WARN [IPC Server handler 0 on default port 40751 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-02T09:18:55,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:55,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:55,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:55,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:55,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:18:55,136 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131133100 with entries=15, filesize=13.26 KB; new WAL /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131135116 2024-12-02T09:18:55,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741857_1040 (size=13591) 2024-12-02T09:18:55,138 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39161:39161)] 2024-12-02T09:18:55,138 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131112776 is not closed yet, will try archiving it next time 2024-12-02T09:18:55,138 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131133100 is not closed yet, will try archiving it next time 2024-12-02T09:18:55,138 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131129079 to hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/oldWALs/7c6d666a4939%2C37839%2C1733131112142.1733131129079 2024-12-02T09:18:55,491 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:55,538 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131112776 is not closed yet, will try archiving it next time 2024-12-02T09:18:55,837 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@3efe6249[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36861, datanodeUuid=35211fc1-d25b-450d-9a57-97363aeea54f, infoPort=39161, infoSecurePort=0, ipcPort=34117, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360):Failed to transfer BP-931683727-172.17.0.3-1733131110360:blk_1073741842_1025 to 127.0.0.1:39375 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:55,837 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@6b6e383c[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36861, datanodeUuid=35211fc1-d25b-450d-9a57-97363aeea54f, infoPort=39161, infoSecurePort=0, ipcPort=34117, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360):Failed to transfer BP-931683727-172.17.0.3-1733131110360:blk_1073741862_1045 to 127.0.0.1:45269 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37839 {}] regionserver.HRegion(8855): Flush requested on ed3ba9f85e4101be92817d08be54655d 2024-12-02T09:18:56,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed3ba9f85e4101be92817d08be54655d 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-12-02T09:18:56,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/.tmp/info/693b423112764c10998d8a49487697b1 is 1079, key is tmprow/info:/1733131136022/Put/seqid=0 2024-12-02T09:18:56,031 WARN [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741878_1061 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:56,031 WARN [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741878_1061 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK], DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK]) is bad. 2024-12-02T09:18:56,031 WARN [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741878_1061 2024-12-02T09:18:56,031 WARN [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK] 2024-12-02T09:18:56,033 WARN [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741879_1062 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:56,033 WARN [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741879_1062 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK], DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK]) is bad. 2024-12-02T09:18:56,033 WARN [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741879_1062 2024-12-02T09:18:56,034 WARN [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK] 2024-12-02T09:18:56,035 WARN [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741880_1063 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:56,035 WARN [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741880_1063 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK], DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK]) is bad. 2024-12-02T09:18:56,035 WARN [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741880_1063 2024-12-02T09:18:56,036 WARN [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK] 2024-12-02T09:18:56,037 WARN [Thread-964 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741881_1064 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:56,037 WARN [Thread-964 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741881_1064 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK], DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]) is bad. 2024-12-02T09:18:56,037 WARN [Thread-964 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741881_1064 2024-12-02T09:18:56,038 WARN [Thread-964 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK] 2024-12-02T09:18:56,038 WARN [IPC Server handler 1 on default port 40751 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-02T09:18:56,039 WARN [IPC Server handler 1 on default port 40751 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-02T09:18:56,039 WARN [IPC Server handler 1 on default port 40751 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-02T09:18:56,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741882_1065 (size=6027) 2024-12-02T09:18:56,288 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:56,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/.tmp/info/693b423112764c10998d8a49487697b1 2024-12-02T09:18:56,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/.tmp/info/693b423112764c10998d8a49487697b1 as hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/693b423112764c10998d8a49487697b1 2024-12-02T09:18:56,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/693b423112764c10998d8a49487697b1, entries=1, sequenceid=55, filesize=5.9 K 2024-12-02T09:18:56,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for ed3ba9f85e4101be92817d08be54655d in 439ms, sequenceid=55, compaction requested=true 2024-12-02T09:18:56,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed3ba9f85e4101be92817d08be54655d: 2024-12-02T09:18:56,462 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=29.3 K, sizeToCheck=16.0 K 2024-12-02T09:18:56,462 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-02T09:18:56,462 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/e8e3c0ec93a84097ad7a6d97d6dac64c because midkey is the same as first or last row 2024-12-02T09:18:56,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ed3ba9f85e4101be92817d08be54655d:info, priority=-2147483648, current under compaction store size is 1 2024-12-02T09:18:56,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:18:56,462 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T09:18:56,464 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 30048 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T09:18:56,464 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.HStore(1541): ed3ba9f85e4101be92817d08be54655d/info is initiating minor compaction (all files) 2024-12-02T09:18:56,464 INFO [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ed3ba9f85e4101be92817d08be54655d/info in TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d. 2024-12-02T09:18:56,464 INFO [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/e8e3c0ec93a84097ad7a6d97d6dac64c, hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/e6e7951f47444803857f432b3a8e048b, hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/693b423112764c10998d8a49487697b1] into tmpdir=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/.tmp, totalSize=29.3 K 2024-12-02T09:18:56,465 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8e3c0ec93a84097ad7a6d97d6dac64c, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733131127108 2024-12-02T09:18:56,465 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6e7951f47444803857f432b3a8e048b, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733131134600 2024-12-02T09:18:56,465 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] compactions.Compactor(225): Compacting 693b423112764c10998d8a49487697b1, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733131136022 2024-12-02T09:18:56,481 INFO [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ed3ba9f85e4101be92817d08be54655d#info#compaction#24 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T09:18:56,482 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/.tmp/info/e8b973ed32d24b37bea74a49fdab50fe is 1080, key is row0002/info:/1733131127108/Put/seqid=0 2024-12-02T09:18:56,483 WARN [Thread-968 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741883_1066 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:56,484 WARN [Thread-968 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741883_1066 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK], DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK]) is bad. 2024-12-02T09:18:56,484 WARN [Thread-968 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741883_1066 2024-12-02T09:18:56,484 WARN [Thread-968 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39375,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK] 2024-12-02T09:18:56,486 WARN [Thread-968 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741884_1067 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:56,486 WARN [Thread-968 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741884_1067 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK], DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK]) is bad. 2024-12-02T09:18:56,486 WARN [Thread-968 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741884_1067 2024-12-02T09:18:56,487 WARN [Thread-968 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45269,DS-237045ba-35fe-4226-80c0-06528c0e18c0,DISK] 2024-12-02T09:18:56,489 WARN [Thread-968 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741885_1068 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:39031 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:56,489 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:50138 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741885_1068] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data6]'}, localName='127.0.0.1:36861', datanodeUuid='35211fc1-d25b-450d-9a57-97363aeea54f', xmitsInProgress=0}:Exception transferring block BP-931683727-172.17.0.3-1733131110360:blk_1073741885_1068 to mirror 127.0.0.1:39031 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:56,489 WARN [Thread-968 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741885_1068 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK], DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK]) is bad. 2024-12-02T09:18:56,489 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:50138 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741885_1068] {}] datanode.BlockReceiver(316): Block 1073741885 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-02T09:18:56,489 WARN [Thread-968 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741885_1068 2024-12-02T09:18:56,489 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:50138 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741885_1068] {}] datanode.DataXceiver(331): 127.0.0.1:36861:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50138 dst: /127.0.0.1:36861 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:56,490 WARN [Thread-968 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK] 2024-12-02T09:18:56,491 WARN [Thread-968 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741886_1069 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:56,491 WARN [Thread-968 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741886_1069 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK], DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]) is bad. 2024-12-02T09:18:56,491 WARN [Thread-968 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741886_1069 2024-12-02T09:18:56,492 WARN [Thread-968 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK] 2024-12-02T09:18:56,493 WARN [IPC Server handler 0 on default port 40751 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-02T09:18:56,493 WARN [IPC Server handler 0 on default port 40751 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-02T09:18:56,493 WARN [IPC Server handler 0 on default port 40751 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-02T09:18:56,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741887_1070 (size=18097) 2024-12-02T09:18:56,837 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@6b6e383c[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36861, datanodeUuid=35211fc1-d25b-450d-9a57-97363aeea54f, infoPort=39161, infoSecurePort=0, ipcPort=34117, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360):Failed to transfer BP-931683727-172.17.0.3-1733131110360:blk_1073741867_1050 to 127.0.0.1:39031 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:56,837 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@3efe6249[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36861, datanodeUuid=35211fc1-d25b-450d-9a57-97363aeea54f, infoPort=39161, infoSecurePort=0, ipcPort=34117, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360):Failed to transfer BP-931683727-172.17.0.3-1733131110360:blk_1073741872_1055 to 127.0.0.1:40489 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:56,904 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/.tmp/info/e8b973ed32d24b37bea74a49fdab50fe as hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/e8b973ed32d24b37bea74a49fdab50fe 2024-12-02T09:18:56,911 INFO [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed3ba9f85e4101be92817d08be54655d/info of ed3ba9f85e4101be92817d08be54655d into e8b973ed32d24b37bea74a49fdab50fe(size=17.7 K), total size for store is 17.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T09:18:56,911 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ed3ba9f85e4101be92817d08be54655d: 2024-12-02T09:18:56,911 INFO [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d., storeName=ed3ba9f85e4101be92817d08be54655d/info, priority=13, startTime=1733131136462; duration=0sec 2024-12-02T09:18:56,911 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K 2024-12-02T09:18:56,911 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-02T09:18:56,911 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/e8b973ed32d24b37bea74a49fdab50fe because midkey is the same as first or last row 2024-12-02T09:18:56,911 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K 2024-12-02T09:18:56,911 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-02T09:18:56,911 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/e8b973ed32d24b37bea74a49fdab50fe because midkey is the same as first or last row 2024-12-02T09:18:56,911 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K 2024-12-02T09:18:56,912 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-02T09:18:56,912 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/e8b973ed32d24b37bea74a49fdab50fe because midkey is the same as first or last row 2024-12-02T09:18:56,912 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:18:56,912 DEBUG [RS:0;7c6d666a4939:37839-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ed3ba9f85e4101be92817d08be54655d:info 2024-12-02T09:18:57,138 WARN [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas. 2024-12-02T09:18:57,138 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:57,253 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:18:57,257 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:18:57,258 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:18:57,258 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:18:57,258 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T09:18:57,259 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@796aacdc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:18:57,259 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3edaa0c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:18:57,361 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@30d4c51a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/java.io.tmpdir/jetty-localhost-37375-hadoop-hdfs-3_4_1-tests_jar-_-any-14075399952334871552/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:18:57,362 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34827478{HTTP/1.1, (http/1.1)}{localhost:37375} 2024-12-02T09:18:57,362 INFO [Time-limited test {}] server.Server(415): Started @133843ms 2024-12-02T09:18:57,363 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T09:18:57,492 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:57,750 WARN [Thread-987 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T09:18:57,760 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x64eb0a761c5ea6db with lease ID 0xe9c6559de3b4cead: from storage DS-99e799c5-14e9-4935-8480-11c3a0722912 node DatanodeRegistration(127.0.0.1:42191, datanodeUuid=900a00a2-9ff5-4c37-9912-5024947f217c, infoPort=44365, infoSecurePort=0, ipcPort=36949, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:18:57,760 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x64eb0a761c5ea6db with lease ID 0xe9c6559de3b4cead: from storage DS-4cedcc62-af50-4a9b-a537-caa0c79444bf node DatanodeRegistration(127.0.0.1:42191, datanodeUuid=900a00a2-9ff5-4c37-9912-5024947f217c, infoPort=44365, infoSecurePort=0, ipcPort=36949, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:18:58,288 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:58,838 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@6b6e383c[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36861, datanodeUuid=35211fc1-d25b-450d-9a57-97363aeea54f, infoPort=39161, infoSecurePort=0, ipcPort=34117, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360):Failed to transfer BP-931683727-172.17.0.3-1733131110360:blk_1073741882_1065 to 127.0.0.1:39031 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:58,838 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@3efe6249[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36861, datanodeUuid=35211fc1-d25b-450d-9a57-97363aeea54f, infoPort=39161, infoSecurePort=0, ipcPort=34117, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360):Failed to transfer BP-931683727-172.17.0.3-1733131110360:blk_1073741857_1040 to 127.0.0.1:45269 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:18:59,139 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:59,492 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:18:59,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42191 is added to blk_1073741887_1070 (size=18097) 2024-12-02T09:19:00,289 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:01,139 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:01,492 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:01,977 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T09:19:02,289 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:02,526 ERROR [FSHLog-0-hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData-prefix:7c6d666a4939,38901,1733131112000 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:02,526 WARN [FSHLog-0-hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData-prefix:7c6d666a4939,38901,1733131112000 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:02,526 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog 7c6d666a4939%2C38901%2C1733131112000:(num 1733131112309) roll requested 2024-12-02T09:19:02,526 INFO [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C38901%2C1733131112000.1733131142526 2024-12-02T09:19:02,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:02,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:02,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:02,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:02,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:02,532 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/WALs/7c6d666a4939,38901,1733131112000/7c6d666a4939%2C38901%2C1733131112000.1733131112309 with entries=54, filesize=26.67 KB; new WAL /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/WALs/7c6d666a4939,38901,1733131112000/7c6d666a4939%2C38901%2C1733131112000.1733131142526 2024-12-02T09:19:02,532 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:02,532 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:02,533 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/WALs/7c6d666a4939,38901,1733131112000/7c6d666a4939%2C38901%2C1733131112000.1733131112309 2024-12-02T09:19:02,533 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44365:44365),(127.0.0.1/127.0.0.1:39161:39161)] 2024-12-02T09:19:02,533 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/WALs/7c6d666a4939,38901,1733131112000/7c6d666a4939%2C38901%2C1733131112000.1733131112309 is not closed yet, will try archiving it next time 2024-12-02T09:19:02,533 WARN [IPC Server handler 3 on default port 40751 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/WALs/7c6d666a4939,38901,1733131112000/7c6d666a4939%2C38901%2C1733131112000.1733131112309 has not been closed. Lease recovery is in progress. RecoveryId = 1072 for block blk_1073741830_1006 2024-12-02T09:19:02,533 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/WALs/7c6d666a4939,38901,1733131112000/7c6d666a4939%2C38901%2C1733131112000.1733131112309 after 0ms 2024-12-02T09:19:03,139 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:03,493 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:05,140 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:05,493 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:06,534 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/WALs/7c6d666a4939,38901,1733131112000/7c6d666a4939%2C38901%2C1733131112000.1733131112309 after 4001ms 2024-12-02T09:19:07,140 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:07,493 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:07,771 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@6695c07 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-931683727-172.17.0.3-1733131110360:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:40489,null,null]) java.net.ConnectException: Call From 7c6d666a4939/172.17.0.3 to localhost:41383 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-12-02T09:19:07,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42191 is added to blk_1073741833_1019 (size=455) 2024-12-02T09:19:08,105 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131112776 to hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/oldWALs/7c6d666a4939%2C37839%2C1733131112142.1733131112776 2024-12-02T09:19:08,106 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131133100 to hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/oldWALs/7c6d666a4939%2C37839%2C1733131112142.1733131133100 2024-12-02T09:19:09,141 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:09,494 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:10,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741833_1019 (size=455) 2024-12-02T09:19:11,091 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C37839%2C1733131112142.1733131151091 2024-12-02T09:19:11,094 WARN [Thread-1019 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741889_1073 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:11,094 WARN [Thread-1019 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741889_1073 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK], DatanodeInfoWithStorage[127.0.0.1:42191,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK]) is bad. 2024-12-02T09:19:11,094 WARN [Thread-1019 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741889_1073 2024-12-02T09:19:11,094 WARN [Thread-1019 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK] 2024-12-02T09:19:11,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:11,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:11,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:11,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:11,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:11,100 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131135116 with entries=13, filesize=12.60 KB; new WAL /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131151091 2024-12-02T09:19:11,101 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44365:44365),(127.0.0.1/127.0.0.1:39161:39161)] 2024-12-02T09:19:11,101 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131135116 is not closed yet, will try archiving it next time 2024-12-02T09:19:11,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741877_1060 (size=12911) 2024-12-02T09:19:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37839 {}] regionserver.HRegion(8855): Flush requested on ed3ba9f85e4101be92817d08be54655d 2024-12-02T09:19:11,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ed3ba9f85e4101be92817d08be54655d 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-12-02T09:19:11,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/.tmp/info/b5135dbdd3344ab5a7afec8dbdf6bb08 is 1080, key is row0013/info:/1733131151102/Put/seqid=0 2024-12-02T09:19:11,112 WARN [Thread-1025 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741891_1075 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:11,112 WARN [Thread-1025 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741891_1075 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK], DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK]) is bad. 2024-12-02T09:19:11,112 WARN [Thread-1025 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741891_1075 2024-12-02T09:19:11,113 WARN [Thread-1025 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK] 2024-12-02T09:19:11,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42191 is added to blk_1073741892_1076 (size=8190) 2024-12-02T09:19:11,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741892_1076 (size=8190) 2024-12-02T09:19:11,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/.tmp/info/b5135dbdd3344ab5a7afec8dbdf6bb08 2024-12-02T09:19:11,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/.tmp/info/b5135dbdd3344ab5a7afec8dbdf6bb08 as hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/b5135dbdd3344ab5a7afec8dbdf6bb08 2024-12-02T09:19:11,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/b5135dbdd3344ab5a7afec8dbdf6bb08, entries=3, sequenceid=66, filesize=8.0 K 2024-12-02T09:19:11,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7527, heapSize ~8.11 KB/8304, currentSize=9.46 KB/9683 for ed3ba9f85e4101be92817d08be54655d in 29ms, sequenceid=66, compaction requested=false 2024-12-02T09:19:11,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ed3ba9f85e4101be92817d08be54655d: 2024-12-02T09:19:11,134 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=25.7 K, sizeToCheck=16.0 K 2024-12-02T09:19:11,134 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-02T09:19:11,134 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/e8b973ed32d24b37bea74a49fdab50fe because midkey is the same as first or last row 2024-12-02T09:19:11,141 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled. 2024-12-02T09:19:11,141 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:11,328 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-02T09:19:11,329 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T09:19:11,329 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T09:19:11,329 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:19:11,329 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:19:11,329 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T09:19:11,329 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-02T09:19:11,329 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=855045927, stopped=false 2024-12-02T09:19:11,329 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7c6d666a4939,38901,1733131112000 2024-12-02T09:19:11,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37839-0x1009a47ed610001, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T09:19:11,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T09:19:11,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x1009a47ed610002, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T09:19:11,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:11,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x1009a47ed610002, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:11,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37839-0x1009a47ed610001, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:11,359 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T09:19:11,359 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T09:19:11,360 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T09:19:11,360 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:19:11,360 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37839-0x1009a47ed610001, quorum=127.0.0.1:51860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:19:11,360 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7c6d666a4939,37839,1733131112142' ***** 2024-12-02T09:19:11,360 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-02T09:19:11,360 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7c6d666a4939,39613,1733131113414' ***** 2024-12-02T09:19:11,360 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-02T09:19:11,360 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:19:11,360 INFO [RS:0;7c6d666a4939:37839 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T09:19:11,360 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39613-0x1009a47ed610002, quorum=127.0.0.1:51860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:19:11,360 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T09:19:11,360 INFO [RS:0;7c6d666a4939:37839 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-02T09:19:11,360 INFO [RS:0;7c6d666a4939:37839 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-02T09:19:11,361 INFO [RS:1;7c6d666a4939:39613 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T09:19:11,361 INFO [RS:0;7c6d666a4939:37839 {}] regionserver.HRegionServer(3091): Received CLOSE for ed3ba9f85e4101be92817d08be54655d 2024-12-02T09:19:11,361 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T09:19:11,361 INFO [RS:1;7c6d666a4939:39613 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-02T09:19:11,361 INFO [RS:0;7c6d666a4939:37839 {}] regionserver.HRegionServer(959): stopping server 7c6d666a4939,37839,1733131112142 2024-12-02T09:19:11,361 INFO [RS:1;7c6d666a4939:39613 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-02T09:19:11,361 INFO [RS:1;7c6d666a4939:39613 {}] regionserver.HRegionServer(959): stopping server 7c6d666a4939,39613,1733131113414 2024-12-02T09:19:11,361 INFO [RS:1;7c6d666a4939:39613 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T09:19:11,361 INFO [RS:0;7c6d666a4939:37839 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T09:19:11,361 INFO [RS:1;7c6d666a4939:39613 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;7c6d666a4939:39613. 2024-12-02T09:19:11,361 INFO [RS:0;7c6d666a4939:37839 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7c6d666a4939:37839. 2024-12-02T09:19:11,361 DEBUG [RS:0;7c6d666a4939:37839 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T09:19:11,361 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ed3ba9f85e4101be92817d08be54655d, disabling compactions & flushes 2024-12-02T09:19:11,361 DEBUG [RS:1;7c6d666a4939:39613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T09:19:11,361 DEBUG [RS:0;7c6d666a4939:37839 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:19:11,361 DEBUG [RS:1;7c6d666a4939:39613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:19:11,361 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d. 2024-12-02T09:19:11,361 INFO [RS:0;7c6d666a4939:37839 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T09:19:11,361 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d. 2024-12-02T09:19:11,361 INFO [RS:1;7c6d666a4939:39613 {}] regionserver.HRegionServer(976): stopping server 7c6d666a4939,39613,1733131113414; all regions closed. 2024-12-02T09:19:11,361 INFO [RS:0;7c6d666a4939:37839 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T09:19:11,361 INFO [RS:0;7c6d666a4939:37839 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T09:19:11,361 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d. after waiting 0 ms 2024-12-02T09:19:11,361 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d. 2024-12-02T09:19:11,361 INFO [RS:0;7c6d666a4939:37839 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-02T09:19:11,362 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing ed3ba9f85e4101be92817d08be54655d 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB 2024-12-02T09:19:11,362 INFO [RS:0;7c6d666a4939:37839 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-02T09:19:11,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:11,362 DEBUG [RS:0;7c6d666a4939:37839 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, ed3ba9f85e4101be92817d08be54655d=TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d.} 2024-12-02T09:19:11,362 DEBUG [RS:0;7c6d666a4939:37839 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, ed3ba9f85e4101be92817d08be54655d 2024-12-02T09:19:11,362 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T09:19:11,362 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T09:19:11,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:11,362 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T09:19:11,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:11,362 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T09:19:11,362 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T09:19:11,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:11,362 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB 2024-12-02T09:19:11,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:11,363 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:11,363 ERROR [FSHLog-0-hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c-prefix:7c6d666a4939,37839,1733131112142.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:11,363 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:11,363 WARN [FSHLog-0-hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c-prefix:7c6d666a4939,37839,1733131112142.meta {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:11,363 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 2024-12-02T09:19:11,363 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 7c6d666a4939%2C37839%2C1733131112142.meta:.meta(num 1733131113174) roll requested 2024-12-02T09:19:11,363 INFO [regionserver/7c6d666a4939:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C37839%2C1733131112142.meta.1733131151363.meta 2024-12-02T09:19:11,363 WARN [IPC Server handler 4 on default port 40751 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 has not been closed. Lease recovery is in progress. RecoveryId = 1077 for block blk_1073741837_1013 2024-12-02T09:19:11,364 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 after 1ms 2024-12-02T09:19:11,368 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/.tmp/info/07da3843ca1e45e4ac72a75b951ce124 is 1080, key is row0015/info:/1733131151106/Put/seqid=0 2024-12-02T09:19:11,370 WARN [Thread-1034 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741894_1079 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:11,370 WARN [Thread-1034 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741894_1079 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK], DatanodeInfoWithStorage[127.0.0.1:42191,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK]) is bad. 2024-12-02T09:19:11,370 WARN [Thread-1034 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741894_1079 2024-12-02T09:19:11,371 WARN [Thread-1034 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK] 2024-12-02T09:19:11,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:11,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:11,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:11,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:11,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:11,377 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131151363.meta 2024-12-02T09:19:11,377 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:11,377 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40489,DS-b5c186eb-2651-4937-820a-5fbdb03c4e97,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:11,377 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta 2024-12-02T09:19:11,377 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39161:39161),(127.0.0.1/127.0.0.1:44365:44365)] 2024-12-02T09:19:11,378 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta is not closed yet, will try archiving it next time 2024-12-02T09:19:11,378 WARN [IPC Server handler 1 on default port 40751 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta has not been closed. Lease recovery is in progress. RecoveryId = 1081 for block blk_1073741834_1010 2024-12-02T09:19:11,378 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta after 1ms 2024-12-02T09:19:11,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741895_1080 (size=14660) 2024-12-02T09:19:11,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42191 is added to blk_1073741895_1080 (size=14660) 2024-12-02T09:19:11,379 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/.tmp/info/07da3843ca1e45e4ac72a75b951ce124 2024-12-02T09:19:11,387 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/.tmp/info/07da3843ca1e45e4ac72a75b951ce124 as hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/07da3843ca1e45e4ac72a75b951ce124 2024-12-02T09:19:11,393 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/07da3843ca1e45e4ac72a75b951ce124, entries=9, sequenceid=78, filesize=14.3 K 2024-12-02T09:19:11,395 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9683, heapSize ~10.36 KB/10608, currentSize=0 B/0 for ed3ba9f85e4101be92817d08be54655d in 34ms, sequenceid=78, compaction requested=true 2024-12-02T09:19:11,396 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/8d0a8c7afa3044419e89d6ab42aa0399, hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/5b8bd58425ca475abbfda4b9fe58edcd, hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/e8e3c0ec93a84097ad7a6d97d6dac64c, hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/0831543502bb4968927c17a3d1b41baf, hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/e6e7951f47444803857f432b3a8e048b, hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/693b423112764c10998d8a49487697b1] to archive 2024-12-02T09:19:11,397 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/hbase/meta/1588230740/.tmp/info/557de64da7784f1cbe24e7eb2aec7dd8 is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d./info:regioninfo/1733131113896/Put/seqid=0 2024-12-02T09:19:11,397 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T09:19:11,398 WARN [Thread-1045 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741896_1082 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:11,399 WARN [Thread-1045 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741896_1082 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK], DatanodeInfoWithStorage[127.0.0.1:36861,DS-45673769-6335-4172-b966-5b5eefbef64f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK]) is bad. 2024-12-02T09:19:11,399 WARN [Thread-1045 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741896_1082 2024-12-02T09:19:11,399 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/8d0a8c7afa3044419e89d6ab42aa0399 to hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/8d0a8c7afa3044419e89d6ab42aa0399 2024-12-02T09:19:11,399 WARN [Thread-1045 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK] 2024-12-02T09:19:11,401 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/5b8bd58425ca475abbfda4b9fe58edcd to hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/5b8bd58425ca475abbfda4b9fe58edcd 2024-12-02T09:19:11,404 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/e8e3c0ec93a84097ad7a6d97d6dac64c to hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/e8e3c0ec93a84097ad7a6d97d6dac64c 2024-12-02T09:19:11,406 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/0831543502bb4968927c17a3d1b41baf to hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/0831543502bb4968927c17a3d1b41baf 2024-12-02T09:19:11,407 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/e6e7951f47444803857f432b3a8e048b to hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/e6e7951f47444803857f432b3a8e048b 2024-12-02T09:19:11,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741897_1083 (size=7089) 2024-12-02T09:19:11,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42191 is added to blk_1073741897_1083 (size=7089) 2024-12-02T09:19:11,408 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/hbase/meta/1588230740/.tmp/info/557de64da7784f1cbe24e7eb2aec7dd8 2024-12-02T09:19:11,409 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/693b423112764c10998d8a49487697b1 to hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/info/693b423112764c10998d8a49487697b1 2024-12-02T09:19:11,409 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=7c6d666a4939:38901 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-12-02T09:19:11,410 WARN [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [8d0a8c7afa3044419e89d6ab42aa0399=10347, 5b8bd58425ca475abbfda4b9fe58edcd=12506, e8e3c0ec93a84097ad7a6d97d6dac64c=17994, 0831543502bb4968927c17a3d1b41baf=6027, e6e7951f47444803857f432b3a8e048b=6027, 693b423112764c10998d8a49487697b1=6027] 2024-12-02T09:19:11,416 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/default/TestLogRolling-testLogRollOnDatanodeDeath/ed3ba9f85e4101be92817d08be54655d/recovered.edits/81.seqid, newMaxSeqId=81, maxSeqId=1 2024-12-02T09:19:11,416 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d. 2024-12-02T09:19:11,416 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ed3ba9f85e4101be92817d08be54655d: Waiting for close lock at 1733131151361Running coprocessor pre-close hooks at 1733131151361Disabling compacts and flushes for region at 1733131151361Disabling writes for close at 1733131151361Obtaining lock to block concurrent updates at 1733131151362 (+1 ms)Preparing flush snapshotting stores in ed3ba9f85e4101be92817d08be54655d at 1733131151362Finished memstore snapshotting TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d., syncing WAL and waiting on mvcc, flushsize=dataSize=9683, getHeapSize=10608, getOffHeapSize=0, getCellsCount=9 at 1733131151362Flushing stores of TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d. at 1733131151363 (+1 ms)Flushing ed3ba9f85e4101be92817d08be54655d/info: creating writer at 1733131151363Flushing ed3ba9f85e4101be92817d08be54655d/info: appending metadata at 1733131151368 (+5 ms)Flushing ed3ba9f85e4101be92817d08be54655d/info: closing flushed file at 1733131151368Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18824ccd: reopening flushed file at 1733131151386 (+18 ms)Finished flush of dataSize ~9.46 KB/9683, heapSize ~10.36 KB/10608, currentSize=0 B/0 for ed3ba9f85e4101be92817d08be54655d in 34ms, sequenceid=78, compaction requested=true at 1733131151395 (+9 ms)Writing region close event to WAL at 1733131151410 (+15 ms)Running coprocessor post-close hooks at 1733131151416 (+6 ms)Closed at 1733131151416 2024-12-02T09:19:11,416 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733131113528.ed3ba9f85e4101be92817d08be54655d. 2024-12-02T09:19:11,431 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/hbase/meta/1588230740/.tmp/ns/32ef9963b9be461c8c846a7492955be2 is 43, key is default/ns:d/1733131113282/Put/seqid=0 2024-12-02T09:19:11,434 WARN [Thread-1052 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741898_1084 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:39031 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:11,434 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:57136 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741898_1084] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data4]'}, localName='127.0.0.1:42191', datanodeUuid='900a00a2-9ff5-4c37-9912-5024947f217c', xmitsInProgress=0}:Exception transferring block BP-931683727-172.17.0.3-1733131110360:blk_1073741898_1084 to mirror 127.0.0.1:39031 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:19:11,434 WARN [Thread-1052 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741898_1084 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42191,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK], DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK]) is bad. 2024-12-02T09:19:11,434 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:57136 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741898_1084] {}] datanode.BlockReceiver(316): Block 1073741898 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-02T09:19:11,434 WARN [Thread-1052 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741898_1084 2024-12-02T09:19:11,434 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:57136 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741898_1084] {}] datanode.DataXceiver(331): 127.0.0.1:42191:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57136 dst: /127.0.0.1:42191 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:19:11,435 WARN [Thread-1052 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK] 2024-12-02T09:19:11,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42191 is added to blk_1073741899_1085 (size=5153) 2024-12-02T09:19:11,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741899_1085 (size=5153) 2024-12-02T09:19:11,489 INFO [regionserver/7c6d666a4939:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-02T09:19:11,489 INFO [regionserver/7c6d666a4939:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-02T09:19:11,492 INFO [regionserver/7c6d666a4939:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T09:19:11,502 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.1733131135116 to hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/oldWALs/7c6d666a4939%2C37839%2C1733131112142.1733131135116 2024-12-02T09:19:11,562 DEBUG [RS:0;7c6d666a4939:37839 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T09:19:11,630 INFO [regionserver/7c6d666a4939:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-02T09:19:11,630 INFO [regionserver/7c6d666a4939:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-02T09:19:11,762 DEBUG [RS:0;7c6d666a4939:37839 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T09:19:11,844 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/hbase/meta/1588230740/.tmp/ns/32ef9963b9be461c8c846a7492955be2 2024-12-02T09:19:11,865 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/hbase/meta/1588230740/.tmp/table/95b7b5972fc74b79932134b75fc28208 is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1733131113907/Put/seqid=0 2024-12-02T09:19:11,868 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:57154 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741900_1086] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data4]'}, localName='127.0.0.1:42191', datanodeUuid='900a00a2-9ff5-4c37-9912-5024947f217c', xmitsInProgress=0}:Exception transferring block BP-931683727-172.17.0.3-1733131110360:blk_1073741900_1086 to mirror 127.0.0.1:39031 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:19:11,868 WARN [Thread-1060 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741900_1086 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:39031 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:11,868 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:57154 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741900_1086] {}] datanode.BlockReceiver(316): Block 1073741900 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-12-02T09:19:11,868 WARN [Thread-1060 {}] hdfs.DataStreamer(1731): Error Recovery for BP-931683727-172.17.0.3-1733131110360:blk_1073741900_1086 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42191,DS-99e799c5-14e9-4935-8480-11c3a0722912,DISK], DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK]) is bad. 2024-12-02T09:19:11,868 WARN [Thread-1060 {}] hdfs.DataStreamer(1850): Abandoning BP-931683727-172.17.0.3-1733131110360:blk_1073741900_1086 2024-12-02T09:19:11,868 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1052569915_22 at /127.0.0.1:57154 [Receiving block BP-931683727-172.17.0.3-1733131110360:blk_1073741900_1086] {}] datanode.DataXceiver(331): 127.0.0.1:42191:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57154 dst: /127.0.0.1:42191 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:19:11,869 WARN [Thread-1060 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39031,DS-217d50f4-1288-443d-99ce-08122b86cee7,DISK] 2024-12-02T09:19:11,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42191 is added to blk_1073741901_1087 (size=5424) 2024-12-02T09:19:11,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741901_1087 (size=5424) 2024-12-02T09:19:11,875 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/hbase/meta/1588230740/.tmp/table/95b7b5972fc74b79932134b75fc28208 2024-12-02T09:19:11,882 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/hbase/meta/1588230740/.tmp/info/557de64da7784f1cbe24e7eb2aec7dd8 as hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/hbase/meta/1588230740/info/557de64da7784f1cbe24e7eb2aec7dd8 2024-12-02T09:19:11,889 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/hbase/meta/1588230740/info/557de64da7784f1cbe24e7eb2aec7dd8, entries=10, sequenceid=11, filesize=6.9 K 2024-12-02T09:19:11,890 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/hbase/meta/1588230740/.tmp/ns/32ef9963b9be461c8c846a7492955be2 as hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/hbase/meta/1588230740/ns/32ef9963b9be461c8c846a7492955be2 2024-12-02T09:19:11,897 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/hbase/meta/1588230740/ns/32ef9963b9be461c8c846a7492955be2, entries=2, sequenceid=11, filesize=5.0 K 2024-12-02T09:19:11,898 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/hbase/meta/1588230740/.tmp/table/95b7b5972fc74b79932134b75fc28208 as hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/hbase/meta/1588230740/table/95b7b5972fc74b79932134b75fc28208 2024-12-02T09:19:11,904 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/hbase/meta/1588230740/table/95b7b5972fc74b79932134b75fc28208, entries=2, sequenceid=11, filesize=5.3 K 2024-12-02T09:19:11,905 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 543ms, sequenceid=11, compaction requested=false 2024-12-02T09:19:11,910 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-02T09:19:11,911 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T09:19:11,911 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T09:19:11,911 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733131151362Running coprocessor pre-close hooks at 1733131151362Disabling compacts and flushes for region at 1733131151362Disabling writes for close at 1733131151362Obtaining lock to block concurrent updates at 1733131151362Preparing flush snapshotting stores in 1588230740 at 1733131151362Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1733131151363 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733131151379 (+16 ms)Flushing 1588230740/info: creating writer at 1733131151379Flushing 1588230740/info: appending metadata at 1733131151396 (+17 ms)Flushing 1588230740/info: closing flushed file at 1733131151396Flushing 1588230740/ns: creating writer at 1733131151414 (+18 ms)Flushing 1588230740/ns: appending metadata at 1733131151431 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1733131151431Flushing 1588230740/table: creating writer at 1733131151850 (+419 ms)Flushing 1588230740/table: appending metadata at 1733131151864 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733131151864Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70ba41f4: reopening flushed file at 1733131151881 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@214bda46: reopening flushed file at 1733131151889 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60e76df0: reopening flushed file at 1733131151897 (+8 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 543ms, sequenceid=11, compaction requested=false at 1733131151905 (+8 ms)Writing region close event to WAL at 1733131151907 (+2 ms)Running coprocessor post-close hooks at 1733131151911 (+4 ms)Closed at 1733131151911 2024-12-02T09:19:11,911 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-02T09:19:11,962 INFO [RS:0;7c6d666a4939:37839 {}] regionserver.HRegionServer(976): stopping server 7c6d666a4939,37839,1733131112142; all regions closed. 2024-12-02T09:19:11,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:11,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:11,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:11,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:11,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:11,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42191 is added to blk_1073741893_1078 (size=825) 2024-12-02T09:19:11,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741893_1078 (size=825) 2024-12-02T09:19:12,632 INFO [regionserver/7c6d666a4939:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T09:19:13,377 INFO [master/7c6d666a4939:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-02T09:19:13,377 INFO [master/7c6d666a4939:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-02T09:19:13,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741836_1012 (size=76) 2024-12-02T09:19:13,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741832_1008 (size=32) 2024-12-02T09:19:14,755 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@6646a348[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:42191, datanodeUuid=900a00a2-9ff5-4c37-9912-5024947f217c, infoPort=44365, infoSecurePort=0, ipcPort=36949, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360):Failed to transfer BP-931683727-172.17.0.3-1733131110360:blk_1073741828_1004 to 127.0.0.1:39031 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:19:14,755 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@37255c3b[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:42191, datanodeUuid=900a00a2-9ff5-4c37-9912-5024947f217c, infoPort=44365, infoSecurePort=0, ipcPort=36949, storageInfo=lv=-57;cid=testClusterID;nsid=2020534138;c=1733131110360):Failed to transfer BP-931683727-172.17.0.3-1733131110360:blk_1073741826_1002 to 127.0.0.1:39031 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:19:15,365 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 after 4002ms 2024-12-02T09:19:15,379 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta after 4002ms 2024-12-02T09:19:16,363 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds" 2024-12-02T09:19:16,365 DEBUG [RS:1;7c6d666a4939:39613 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/oldWALs 2024-12-02T09:19:16,365 INFO [RS:1;7c6d666a4939:39613 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7c6d666a4939%2C39613%2C1733131113414:(num 1733131113632) 2024-12-02T09:19:16,365 DEBUG [RS:1;7c6d666a4939:39613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:19:16,365 INFO [RS:1;7c6d666a4939:39613 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T09:19:16,365 INFO [RS:1;7c6d666a4939:39613 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T09:19:16,365 INFO [RS:1;7c6d666a4939:39613 {}] hbase.ChoreService(370): Chore service for: regionserver/7c6d666a4939:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-02T09:19:16,365 INFO [RS:1;7c6d666a4939:39613 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T09:19:16,365 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T09:19:16,365 INFO [RS:1;7c6d666a4939:39613 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T09:19:16,365 INFO [RS:1;7c6d666a4939:39613 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T09:19:16,365 INFO [RS:1;7c6d666a4939:39613 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T09:19:16,366 INFO [RS:1;7c6d666a4939:39613 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39613 2024-12-02T09:19:16,369 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor105.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:16,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x1009a47ed610002, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7c6d666a4939,39613,1733131113414 2024-12-02T09:19:16,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T09:19:16,401 INFO [RS:1;7c6d666a4939:39613 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T09:19:16,402 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7c6d666a4939,39613,1733131113414] 2024-12-02T09:19:16,417 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7c6d666a4939,39613,1733131113414 already deleted, retry=false 2024-12-02T09:19:16,417 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7c6d666a4939,39613,1733131113414 expired; onlineServers=1 2024-12-02T09:19:16,417 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:16,428 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:16,428 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:16,429 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:16,429 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:16,430 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:16,437 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:16,437 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:16,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x1009a47ed610002, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:19:16,509 INFO [RS:1;7c6d666a4939:39613 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T09:19:16,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39613-0x1009a47ed610002, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:19:16,509 INFO [RS:1;7c6d666a4939:39613 {}] regionserver.HRegionServer(1031): Exiting; stopping=7c6d666a4939,39613,1733131113414; zookeeper connection closed. 2024-12-02T09:19:16,510 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@46a14dd2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@46a14dd2 2024-12-02T09:19:16,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741827_1003 (size=196) 2024-12-02T09:19:16,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741829_1005 (size=34) 2024-12-02T09:19:16,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42191 is added to blk_1073741877_1060 (size=12911) 2024-12-02T09:19:16,940 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-02T09:19:16,958 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:16,959 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:16,960 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:16,960 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:16,960 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:16,963 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:16,963 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:16,964 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds" 2024-12-02T09:19:16,965 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:16,967 DEBUG [RS:0;7c6d666a4939:37839 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/oldWALs 2024-12-02T09:19:16,967 INFO [RS:0;7c6d666a4939:37839 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7c6d666a4939%2C37839%2C1733131112142.meta:.meta(num 1733131151363) 2024-12-02T09:19:16,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:16,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:16,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:16,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:16,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:16,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741890_1074 (size=14682) 2024-12-02T09:19:16,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42191 is added to blk_1073741890_1074 (size=14682) 2024-12-02T09:19:16,972 DEBUG [RS:0;7c6d666a4939:37839 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/oldWALs 2024-12-02T09:19:16,972 INFO [RS:0;7c6d666a4939:37839 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7c6d666a4939%2C37839%2C1733131112142:(num 1733131151091) 2024-12-02T09:19:16,972 DEBUG [RS:0;7c6d666a4939:37839 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:19:16,972 INFO [RS:0;7c6d666a4939:37839 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T09:19:16,972 INFO [RS:0;7c6d666a4939:37839 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T09:19:16,972 INFO [RS:0;7c6d666a4939:37839 {}] hbase.ChoreService(370): Chore service for: regionserver/7c6d666a4939:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-02T09:19:16,973 INFO [RS:0;7c6d666a4939:37839 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T09:19:16,973 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T09:19:16,973 INFO [RS:0;7c6d666a4939:37839 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37839 2024-12-02T09:19:17,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37839-0x1009a47ed610001, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7c6d666a4939,37839,1733131112142 2024-12-02T09:19:17,000 INFO [RS:0;7c6d666a4939:37839 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T09:19:17,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T09:19:17,009 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7c6d666a4939,37839,1733131112142] 2024-12-02T09:19:17,017 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7c6d666a4939,37839,1733131112142 already deleted, retry=false 2024-12-02T09:19:17,017 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7c6d666a4939,37839,1733131112142 expired; onlineServers=0 2024-12-02T09:19:17,017 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7c6d666a4939,38901,1733131112000' ***** 2024-12-02T09:19:17,017 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-02T09:19:17,017 INFO [M:0;7c6d666a4939:38901 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T09:19:17,017 INFO [M:0;7c6d666a4939:38901 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T09:19:17,018 DEBUG [M:0;7c6d666a4939:38901 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-02T09:19:17,018 DEBUG [M:0;7c6d666a4939:38901 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-02T09:19:17,018 DEBUG [master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.small.0-1733131112526 {}] cleaner.HFileCleaner(306): Exit Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.small.0-1733131112526,5,FailOnTimeoutGroup] 2024-12-02T09:19:17,018 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-02T09:19:17,018 DEBUG [master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.large.0-1733131112526 {}] cleaner.HFileCleaner(306): Exit Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.large.0-1733131112526,5,FailOnTimeoutGroup] 2024-12-02T09:19:17,018 INFO [M:0;7c6d666a4939:38901 {}] hbase.ChoreService(370): Chore service for: master/7c6d666a4939:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-02T09:19:17,018 INFO [M:0;7c6d666a4939:38901 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T09:19:17,018 DEBUG [M:0;7c6d666a4939:38901 {}] master.HMaster(1795): Stopping service threads 2024-12-02T09:19:17,019 INFO [M:0;7c6d666a4939:38901 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-02T09:19:17,019 INFO [M:0;7c6d666a4939:38901 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T09:19:17,019 INFO [M:0;7c6d666a4939:38901 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T09:19:17,019 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-02T09:19:17,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-02T09:19:17,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:17,026 DEBUG [M:0;7c6d666a4939:38901 {}] zookeeper.ZKUtil(347): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-02T09:19:17,026 WARN [M:0;7c6d666a4939:38901 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-02T09:19:17,027 INFO [M:0;7c6d666a4939:38901 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/.lastflushedseqids 2024-12-02T09:19:17,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741902_1088 (size=130) 2024-12-02T09:19:17,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42191 is added to blk_1073741902_1088 (size=130) 2024-12-02T09:19:17,038 INFO [M:0;7c6d666a4939:38901 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T09:19:17,039 INFO [M:0;7c6d666a4939:38901 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-02T09:19:17,039 DEBUG [M:0;7c6d666a4939:38901 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T09:19:17,039 INFO [M:0;7c6d666a4939:38901 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:19:17,039 DEBUG [M:0;7c6d666a4939:38901 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:19:17,039 DEBUG [M:0;7c6d666a4939:38901 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T09:19:17,039 DEBUG [M:0;7c6d666a4939:38901 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:19:17,039 INFO [M:0;7c6d666a4939:38901 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.25 KB heapSize=29.49 KB 2024-12-02T09:19:17,056 DEBUG [M:0;7c6d666a4939:38901 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/25c847994f2a4cb68850cf2008cb5b91 is 82, key is hbase:meta,,1/info:regioninfo/1733131113228/Put/seqid=0 2024-12-02T09:19:17,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42191 is added to blk_1073741903_1089 (size=5672) 2024-12-02T09:19:17,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741903_1089 (size=5672) 2024-12-02T09:19:17,062 INFO [M:0;7c6d666a4939:38901 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/25c847994f2a4cb68850cf2008cb5b91 2024-12-02T09:19:17,087 DEBUG [M:0;7c6d666a4939:38901 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f2d0ba94cf5e4cb1a66f71bca6fd4976 is 774, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733131113912/Put/seqid=0 2024-12-02T09:19:17,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741904_1090 (size=6255) 2024-12-02T09:19:17,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42191 is added to blk_1073741904_1090 (size=6255) 2024-12-02T09:19:17,093 INFO [M:0;7c6d666a4939:38901 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.58 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f2d0ba94cf5e4cb1a66f71bca6fd4976 2024-12-02T09:19:17,098 INFO [M:0;7c6d666a4939:38901 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f2d0ba94cf5e4cb1a66f71bca6fd4976 2024-12-02T09:19:17,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37839-0x1009a47ed610001, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:19:17,109 INFO [RS:0;7c6d666a4939:37839 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T09:19:17,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37839-0x1009a47ed610001, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:19:17,109 INFO [RS:0;7c6d666a4939:37839 {}] regionserver.HRegionServer(1031): Exiting; stopping=7c6d666a4939,37839,1733131112142; zookeeper connection closed. 2024-12-02T09:19:17,109 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@760b35d9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@760b35d9 2024-12-02T09:19:17,109 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-02T09:19:17,112 DEBUG [M:0;7c6d666a4939:38901 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/24828bb0cdd14a82b816aed068fec417 is 69, key is 7c6d666a4939,37839,1733131112142/rs:state/1733131112608/Put/seqid=0 2024-12-02T09:19:17,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741905_1091 (size=5224) 2024-12-02T09:19:17,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42191 is added to blk_1073741905_1091 (size=5224) 2024-12-02T09:19:17,118 INFO [M:0;7c6d666a4939:38901 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/24828bb0cdd14a82b816aed068fec417 2024-12-02T09:19:17,140 DEBUG [M:0;7c6d666a4939:38901 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b7549844f0634aa68c199a3fe7f43ad3 is 52, key is load_balancer_on/state:d/1733131113391/Put/seqid=0 2024-12-02T09:19:17,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741906_1092 (size=5056) 2024-12-02T09:19:17,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42191 is added to blk_1073741906_1092 (size=5056) 2024-12-02T09:19:17,145 INFO [M:0;7c6d666a4939:38901 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b7549844f0634aa68c199a3fe7f43ad3 2024-12-02T09:19:17,151 DEBUG [M:0;7c6d666a4939:38901 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/25c847994f2a4cb68850cf2008cb5b91 as hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/25c847994f2a4cb68850cf2008cb5b91 2024-12-02T09:19:17,156 INFO [M:0;7c6d666a4939:38901 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/25c847994f2a4cb68850cf2008cb5b91, entries=8, sequenceid=60, filesize=5.5 K 2024-12-02T09:19:17,157 DEBUG [M:0;7c6d666a4939:38901 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f2d0ba94cf5e4cb1a66f71bca6fd4976 as hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f2d0ba94cf5e4cb1a66f71bca6fd4976 2024-12-02T09:19:17,163 INFO [M:0;7c6d666a4939:38901 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f2d0ba94cf5e4cb1a66f71bca6fd4976 2024-12-02T09:19:17,163 INFO [M:0;7c6d666a4939:38901 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f2d0ba94cf5e4cb1a66f71bca6fd4976, entries=6, sequenceid=60, filesize=6.1 K 2024-12-02T09:19:17,164 DEBUG [M:0;7c6d666a4939:38901 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/24828bb0cdd14a82b816aed068fec417 as hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/24828bb0cdd14a82b816aed068fec417 2024-12-02T09:19:17,170 INFO [M:0;7c6d666a4939:38901 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/24828bb0cdd14a82b816aed068fec417, entries=2, sequenceid=60, filesize=5.1 K 2024-12-02T09:19:17,171 DEBUG [M:0;7c6d666a4939:38901 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b7549844f0634aa68c199a3fe7f43ad3 as hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/b7549844f0634aa68c199a3fe7f43ad3 2024-12-02T09:19:17,177 INFO [M:0;7c6d666a4939:38901 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/b7549844f0634aa68c199a3fe7f43ad3, entries=1, sequenceid=60, filesize=4.9 K 2024-12-02T09:19:17,178 INFO [M:0;7c6d666a4939:38901 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 139ms, sequenceid=60, compaction requested=false 2024-12-02T09:19:17,179 INFO [M:0;7c6d666a4939:38901 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:19:17,179 DEBUG [M:0;7c6d666a4939:38901 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733131157039Disabling compacts and flushes for region at 1733131157039Disabling writes for close at 1733131157039Obtaining lock to block concurrent updates at 1733131157039Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733131157039Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23805, getHeapSize=30136, getOffHeapSize=0, getCellsCount=71 at 1733131157039Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733131157040 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733131157040Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733131157056 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733131157056Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733131157066 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733131157086 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733131157086Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733131157098 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733131157112 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733131157112Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733131157124 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733131157139 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733131157139Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@82ed50a: reopening flushed file at 1733131157150 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fda821f: reopening flushed file at 1733131157156 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a51bbe9: reopening flushed file at 1733131157163 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44b6ed90: reopening flushed file at 1733131157170 (+7 ms)Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 139ms, sequenceid=60, compaction requested=false at 1733131157178 (+8 ms)Writing region close event to WAL at 1733131157179 (+1 ms)Closed at 1733131157179 2024-12-02T09:19:17,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:17,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:17,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:17,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:17,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:17,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741888_1071 (size=1045) 2024-12-02T09:19:17,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42191 is added to blk_1073741888_1071 (size=1045) 2024-12-02T09:19:17,370 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:17,380 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:17,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36861 is added to blk_1073741825_1001 (size=7) 2024-12-02T09:19:17,776 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@522505f5 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-931683727-172.17.0.3-1733131110360:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:40489,null,null]) java.net.ConnectException: Call From 7c6d666a4939/172.17.0.3 to localhost:41383 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-12-02T09:19:18,371 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:18,381 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:18,546 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/WALs/7c6d666a4939,38901,1733131112000/7c6d666a4939%2C38901%2C1733131112000.1733131112309 to hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/oldWALs/7c6d666a4939%2C38901%2C1733131112000.1733131112309 2024-12-02T09:19:18,553 INFO [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/MasterData/oldWALs/7c6d666a4939%2C38901%2C1733131112000.1733131112309 to hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/oldWALs/7c6d666a4939%2C38901%2C1733131112000.1733131112309$masterlocalwal$ 2024-12-02T09:19:18,554 INFO [M:0;7c6d666a4939:38901 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-02T09:19:18,554 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T09:19:18,554 INFO [M:0;7c6d666a4939:38901 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38901 2024-12-02T09:19:18,554 INFO [M:0;7c6d666a4939:38901 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T09:19:18,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:19:18,685 INFO [M:0;7c6d666a4939:38901 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T09:19:18,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38901-0x1009a47ed610000, quorum=127.0.0.1:51860, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:19:18,691 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@30d4c51a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:19:18,692 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34827478{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:19:18,692 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:19:18,692 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3edaa0c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:19:18,693 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@796aacdc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/hadoop.log.dir/,STOPPED} 2024-12-02T09:19:18,694 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T09:19:18,694 WARN [BP-931683727-172.17.0.3-1733131110360 heartbeating to localhost/127.0.0.1:40751 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T09:19:18,694 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T09:19:18,694 WARN [BP-931683727-172.17.0.3-1733131110360 heartbeating to localhost/127.0.0.1:40751 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-931683727-172.17.0.3-1733131110360 (Datanode Uuid 900a00a2-9ff5-4c37-9912-5024947f217c) service to localhost/127.0.0.1:40751 2024-12-02T09:19:18,694 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@2a835b7b {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-931683727-172.17.0.3-1733131110360:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:40489,null,null]) java.io.InterruptedIOException: DestHost:destPort localhost:41383 , LocalHost:localPort 7c6d666a4939/172.17.0.3:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-12-02T09:19:18,695 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@2a835b7b {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-931683727-172.17.0.3-1733131110360:blk_1073741837_1013; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:42191,null,null], DatanodeInfoWithStorage[127.0.0.1:40489,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: No block pool offer service for bpid=BP-931683727-172.17.0.3-1733131110360 2024-12-02T09:19:18,695 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@2a835b7b {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-931683727-172.17.0.3-1733131110360:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:40489,null,null]) java.io.IOException: No block pool offer service for bpid=BP-931683727-172.17.0.3-1733131110360 at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:19:18,695 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@2a835b7b {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-931683727-172.17.0.3-1733131110360:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:42191,null,null]) java.io.IOException: No block pool offer service for bpid=BP-931683727-172.17.0.3-1733131110360 at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:19:18,695 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data3/current/BP-931683727-172.17.0.3-1733131110360 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:19:18,695 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@2a835b7b {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-931683727-172.17.0.3-1733131110360:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:40489,null,null], DatanodeInfoWithStorage[127.0.0.1:42191,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-931683727-172.17.0.3-1733131110360:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:40489,null,null], DatanodeInfoWithStorage[127.0.0.1:42191,null,null]] 2024-12-02T09:19:18,695 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data4/current/BP-931683727-172.17.0.3-1733131110360 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:19:18,696 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T09:19:18,700 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5e1eaefc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:19:18,700 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@64b89ed5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:19:18,700 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:19:18,700 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3abd4c2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:19:18,701 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a9d9bc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/hadoop.log.dir/,STOPPED} 2024-12-02T09:19:18,702 WARN [BP-931683727-172.17.0.3-1733131110360 heartbeating to localhost/127.0.0.1:40751 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T09:19:18,702 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T09:19:18,702 WARN [BP-931683727-172.17.0.3-1733131110360 heartbeating to localhost/127.0.0.1:40751 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-931683727-172.17.0.3-1733131110360 (Datanode Uuid 35211fc1-d25b-450d-9a57-97363aeea54f) service to localhost/127.0.0.1:40751 2024-12-02T09:19:18,702 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T09:19:18,703 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data5/current/BP-931683727-172.17.0.3-1733131110360 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:19:18,703 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/cluster_d8ca8af9-bacf-d29e-b59a-044ab5ad0ab3/data/data6/current/BP-931683727-172.17.0.3-1733131110360 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:19:18,703 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T09:19:18,708 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2e195dbd{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T09:19:18,709 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5d9b1613{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:19:18,709 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:19:18,709 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@20aa2ea7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:19:18,709 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3b918d2a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/hadoop.log.dir/,STOPPED} 2024-12-02T09:19:18,716 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-02T09:19:18,744 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-02T09:19:18,751 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=154 (was 78) Potentially hanging thread: nioEventLoopGroup-14-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007fc3bcbefdc8.run(Unknown Source) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:40751 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:40751 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-14-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:40751 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:41123 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: nioEventLoopGroup-15-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-21-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-15-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40751 from jenkins.hfs.3 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40751 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007fc3bcbefdc8.run(Unknown Source) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40751 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:41123 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'DataNode' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40751 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40751 from jenkins.hfs.2 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-21-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-14-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-21-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-20-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40751 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-20-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40751 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-20-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-15-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40751 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=450 (was 405) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=227 (was 236), ProcessCount=11 (was 11), AvailableMemoryMB=1009 (was 1601) 2024-12-02T09:19:18,757 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=154, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=227, ProcessCount=11, AvailableMemoryMB=1009 2024-12-02T09:19:18,757 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-02T09:19:18,757 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/hadoop.log.dir so I do NOT create it in target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142 2024-12-02T09:19:18,757 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dd63e178-9b6a-fc6e-e26b-63d494ad67e2/hadoop.tmp.dir so I do NOT create it in target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142 2024-12-02T09:19:18,757 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/cluster_efcf4126-ea13-2607-d029-835753d214be, deleteOnExit=true 2024-12-02T09:19:18,757 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-02T09:19:18,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/test.cache.data in system properties and HBase conf 2024-12-02T09:19:18,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/hadoop.tmp.dir in system properties and HBase conf 2024-12-02T09:19:18,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/hadoop.log.dir in system properties and HBase conf 2024-12-02T09:19:18,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-02T09:19:18,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-02T09:19:18,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-02T09:19:18,758 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-02T09:19:18,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-02T09:19:18,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-02T09:19:18,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-02T09:19:18,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T09:19:18,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-02T09:19:18,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-02T09:19:18,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T09:19:18,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T09:19:18,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-02T09:19:18,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/nfs.dump.dir in system properties and HBase conf 2024-12-02T09:19:18,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/java.io.tmpdir in system properties and HBase conf 2024-12-02T09:19:18,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T09:19:18,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-02T09:19:18,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-02T09:19:18,770 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-02T09:19:18,979 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:19:18,983 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:19:18,984 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:19:18,984 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:19:18,984 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T09:19:18,985 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:19:18,985 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@341f9f9e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:19:18,985 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7091f2a1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:19:19,075 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1a48749e{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/java.io.tmpdir/jetty-localhost-39439-hadoop-hdfs-3_4_1-tests_jar-_-any-10242248807763531453/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T09:19:19,076 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@218ef558{HTTP/1.1, (http/1.1)}{localhost:39439} 2024-12-02T09:19:19,076 INFO [Time-limited test {}] server.Server(415): Started @155556ms 2024-12-02T09:19:19,086 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-02T09:19:19,291 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:19:19,296 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:19:19,297 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:19:19,297 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:19:19,297 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T09:19:19,298 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4659fe2a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:19:19,298 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7992aa88{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:19:19,372 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:19,381 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:19,397 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@54dbaae8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/java.io.tmpdir/jetty-localhost-38995-hadoop-hdfs-3_4_1-tests_jar-_-any-12809399528676419891/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:19:19,398 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2b27dfb0{HTTP/1.1, (http/1.1)}{localhost:38995} 2024-12-02T09:19:19,398 INFO [Time-limited test {}] server.Server(415): Started @155879ms 2024-12-02T09:19:19,399 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T09:19:19,427 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:19:19,430 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:19:19,431 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:19:19,431 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:19:19,431 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T09:19:19,432 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@36974255{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:19:19,432 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@126bd190{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:19:19,528 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@386ed495{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/java.io.tmpdir/jetty-localhost-37185-hadoop-hdfs-3_4_1-tests_jar-_-any-5526345575238406345/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:19:19,528 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@a4361c3{HTTP/1.1, (http/1.1)}{localhost:37185} 2024-12-02T09:19:19,529 INFO [Time-limited test {}] server.Server(415): Started @156009ms 2024-12-02T09:19:19,530 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T09:19:19,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath 2024-12-02T09:19:19,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T09:19:19,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-02T09:19:19,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-02T09:19:20,148 WARN [Thread-1198 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/cluster_efcf4126-ea13-2607-d029-835753d214be/data/data1/current/BP-1055095783-172.17.0.3-1733131158781/current, will proceed with Du for space computation calculation, 2024-12-02T09:19:20,148 WARN [Thread-1199 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/cluster_efcf4126-ea13-2607-d029-835753d214be/data/data2/current/BP-1055095783-172.17.0.3-1733131158781/current, will proceed with Du for space computation calculation, 2024-12-02T09:19:20,166 WARN [Thread-1162 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T09:19:20,168 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcc0e9a2746498886 with lease ID 0xb843bc2671f9f045: Processing first storage report for DS-a4e5fdd9-9985-482d-a20b-84f74dd61cee from datanode DatanodeRegistration(127.0.0.1:43343, datanodeUuid=d63e8879-27d2-488b-8f03-7cddea8478b9, infoPort=36145, infoSecurePort=0, ipcPort=35669, storageInfo=lv=-57;cid=testClusterID;nsid=2056264030;c=1733131158781) 2024-12-02T09:19:20,168 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcc0e9a2746498886 with lease ID 0xb843bc2671f9f045: from storage DS-a4e5fdd9-9985-482d-a20b-84f74dd61cee node DatanodeRegistration(127.0.0.1:43343, datanodeUuid=d63e8879-27d2-488b-8f03-7cddea8478b9, infoPort=36145, infoSecurePort=0, ipcPort=35669, storageInfo=lv=-57;cid=testClusterID;nsid=2056264030;c=1733131158781), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:19:20,168 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcc0e9a2746498886 with lease ID 0xb843bc2671f9f045: Processing first storage report for DS-9300c2e6-e373-4ddf-b55d-6512e573e650 from datanode DatanodeRegistration(127.0.0.1:43343, datanodeUuid=d63e8879-27d2-488b-8f03-7cddea8478b9, infoPort=36145, infoSecurePort=0, ipcPort=35669, storageInfo=lv=-57;cid=testClusterID;nsid=2056264030;c=1733131158781) 2024-12-02T09:19:20,168 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcc0e9a2746498886 with lease ID 0xb843bc2671f9f045: from storage DS-9300c2e6-e373-4ddf-b55d-6512e573e650 node DatanodeRegistration(127.0.0.1:43343, datanodeUuid=d63e8879-27d2-488b-8f03-7cddea8478b9, infoPort=36145, infoSecurePort=0, ipcPort=35669, storageInfo=lv=-57;cid=testClusterID;nsid=2056264030;c=1733131158781), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:19:20,242 WARN [Thread-1210 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/cluster_efcf4126-ea13-2607-d029-835753d214be/data/data4/current/BP-1055095783-172.17.0.3-1733131158781/current, will proceed with Du for space computation calculation, 2024-12-02T09:19:20,242 WARN [Thread-1209 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/cluster_efcf4126-ea13-2607-d029-835753d214be/data/data3/current/BP-1055095783-172.17.0.3-1733131158781/current, will proceed with Du for space computation calculation, 2024-12-02T09:19:20,259 WARN [Thread-1185 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T09:19:20,261 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfdbaf1049bd45a2 with lease ID 0xb843bc2671f9f046: Processing first storage report for DS-62cbca48-2a31-4ea9-965d-d8668d707202 from datanode DatanodeRegistration(127.0.0.1:36349, datanodeUuid=0f0f13f0-2e6d-469b-b429-2ecb49eeb705, infoPort=43537, infoSecurePort=0, ipcPort=42127, storageInfo=lv=-57;cid=testClusterID;nsid=2056264030;c=1733131158781) 2024-12-02T09:19:20,261 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfdbaf1049bd45a2 with lease ID 0xb843bc2671f9f046: from storage DS-62cbca48-2a31-4ea9-965d-d8668d707202 node DatanodeRegistration(127.0.0.1:36349, datanodeUuid=0f0f13f0-2e6d-469b-b429-2ecb49eeb705, infoPort=43537, infoSecurePort=0, ipcPort=42127, storageInfo=lv=-57;cid=testClusterID;nsid=2056264030;c=1733131158781), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:19:20,261 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfdbaf1049bd45a2 with lease ID 0xb843bc2671f9f046: Processing first storage report for DS-b0899ce4-10e2-4475-a49b-555d1d2e1329 from datanode DatanodeRegistration(127.0.0.1:36349, datanodeUuid=0f0f13f0-2e6d-469b-b429-2ecb49eeb705, infoPort=43537, infoSecurePort=0, ipcPort=42127, storageInfo=lv=-57;cid=testClusterID;nsid=2056264030;c=1733131158781) 2024-12-02T09:19:20,261 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfdbaf1049bd45a2 with lease ID 0xb843bc2671f9f046: from storage DS-b0899ce4-10e2-4475-a49b-555d1d2e1329 node DatanodeRegistration(127.0.0.1:36349, datanodeUuid=0f0f13f0-2e6d-469b-b429-2ecb49eeb705, infoPort=43537, infoSecurePort=0, ipcPort=42127, storageInfo=lv=-57;cid=testClusterID;nsid=2056264030;c=1733131158781), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:19:20,366 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142 2024-12-02T09:19:20,369 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/cluster_efcf4126-ea13-2607-d029-835753d214be/zookeeper_0, clientPort=50585, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/cluster_efcf4126-ea13-2607-d029-835753d214be/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/cluster_efcf4126-ea13-2607-d029-835753d214be/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-02T09:19:20,370 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50585 2024-12-02T09:19:20,370 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:19:20,372 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:19:20,372 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:20,382 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:20,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36349 is added to blk_1073741825_1001 (size=7) 2024-12-02T09:19:20,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43343 is added to blk_1073741825_1001 (size=7) 2024-12-02T09:19:20,385 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d with version=8 2024-12-02T09:19:20,386 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/hbase-staging 2024-12-02T09:19:20,388 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7c6d666a4939:0 server-side Connection retries=45 2024-12-02T09:19:20,388 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:19:20,388 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T09:19:20,388 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T09:19:20,388 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:19:20,388 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T09:19:20,388 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-02T09:19:20,388 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T09:19:20,389 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42359 2024-12-02T09:19:20,391 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42359 connecting to ZooKeeper ensemble=127.0.0.1:50585 2024-12-02T09:19:20,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:423590x0, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T09:19:20,452 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42359-0x1009a48aa650000 connected 2024-12-02T09:19:20,523 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:19:20,525 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:19:20,526 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:19:20,527 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d, hbase.cluster.distributed=false 2024-12-02T09:19:20,528 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T09:19:20,528 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42359 2024-12-02T09:19:20,528 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42359 2024-12-02T09:19:20,529 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42359 2024-12-02T09:19:20,529 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42359 2024-12-02T09:19:20,529 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42359 2024-12-02T09:19:20,544 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7c6d666a4939:0 server-side Connection retries=45 2024-12-02T09:19:20,544 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:19:20,544 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T09:19:20,544 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T09:19:20,544 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:19:20,544 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T09:19:20,544 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T09:19:20,544 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T09:19:20,545 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38235 2024-12-02T09:19:20,546 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38235 connecting to ZooKeeper ensemble=127.0.0.1:50585 2024-12-02T09:19:20,546 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:19:20,548 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:19:20,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:382350x0, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T09:19:20,557 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38235-0x1009a48aa650001 connected 2024-12-02T09:19:20,557 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38235-0x1009a48aa650001, quorum=127.0.0.1:50585, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:19:20,557 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T09:19:20,558 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T09:19:20,558 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38235-0x1009a48aa650001, quorum=127.0.0.1:50585, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T09:19:20,559 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38235-0x1009a48aa650001, quorum=127.0.0.1:50585, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T09:19:20,560 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38235 2024-12-02T09:19:20,560 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38235 2024-12-02T09:19:20,560 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38235 2024-12-02T09:19:20,560 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38235 2024-12-02T09:19:20,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38235 2024-12-02T09:19:20,573 DEBUG [M:0;7c6d666a4939:42359 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7c6d666a4939:42359 2024-12-02T09:19:20,573 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7c6d666a4939,42359,1733131160387 2024-12-02T09:19:20,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:19:20,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38235-0x1009a48aa650001, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:19:20,582 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7c6d666a4939,42359,1733131160387 2024-12-02T09:19:20,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:20,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38235-0x1009a48aa650001, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T09:19:20,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38235-0x1009a48aa650001, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:20,590 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-02T09:19:20,591 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7c6d666a4939,42359,1733131160387 from backup master directory 2024-12-02T09:19:20,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7c6d666a4939,42359,1733131160387 2024-12-02T09:19:20,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38235-0x1009a48aa650001, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:19:20,598 WARN [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T09:19:20,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:19:20,598 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7c6d666a4939,42359,1733131160387 2024-12-02T09:19:20,602 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/hbase.id] with ID: 1b0443a7-4809-48bd-8e0e-09f20387cb0e 2024-12-02T09:19:20,602 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/.tmp/hbase.id 2024-12-02T09:19:20,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43343 is added to blk_1073741826_1002 (size=42) 2024-12-02T09:19:20,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36349 is added to blk_1073741826_1002 (size=42) 2024-12-02T09:19:20,609 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/.tmp/hbase.id]:[hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/hbase.id] 2024-12-02T09:19:20,619 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:19:20,619 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-02T09:19:20,620 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-12-02T09:19:20,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:20,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38235-0x1009a48aa650001, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:20,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36349 is added to blk_1073741827_1003 (size=196) 2024-12-02T09:19:20,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43343 is added to blk_1073741827_1003 (size=196) 2024-12-02T09:19:21,044 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T09:19:21,047 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-02T09:19:21,048 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-02T09:19:21,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43343 is added to blk_1073741828_1004 (size=1189) 2024-12-02T09:19:21,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36349 is added to blk_1073741828_1004 (size=1189) 2024-12-02T09:19:21,057 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/data/master/store 2024-12-02T09:19:21,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43343 is added to blk_1073741829_1005 (size=34) 2024-12-02T09:19:21,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36349 is added to blk_1073741829_1005 (size=34) 2024-12-02T09:19:21,065 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:19:21,065 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T09:19:21,065 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:19:21,065 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:19:21,065 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T09:19:21,065 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:19:21,065 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:19:21,065 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733131161065Disabling compacts and flushes for region at 1733131161065Disabling writes for close at 1733131161065Writing region close event to WAL at 1733131161065Closed at 1733131161065 2024-12-02T09:19:21,066 WARN [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/data/master/store/.initializing 2024-12-02T09:19:21,066 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/WALs/7c6d666a4939,42359,1733131160387 2024-12-02T09:19:21,069 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c6d666a4939%2C42359%2C1733131160387, suffix=, logDir=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/WALs/7c6d666a4939,42359,1733131160387, archiveDir=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/oldWALs, maxLogs=10 2024-12-02T09:19:21,070 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C42359%2C1733131160387.1733131161070 2024-12-02T09:19:21,075 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/WALs/7c6d666a4939,42359,1733131160387/7c6d666a4939%2C42359%2C1733131160387.1733131161070 2024-12-02T09:19:21,085 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36145:36145),(127.0.0.1/127.0.0.1:43537:43537)] 2024-12-02T09:19:21,091 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-02T09:19:21,091 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:19:21,091 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:19:21,091 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:19:21,093 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:19:21,095 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-02T09:19:21,095 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:21,095 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:19:21,095 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:19:21,096 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-02T09:19:21,096 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:21,097 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:19:21,097 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:19:21,098 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-02T09:19:21,098 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:21,099 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:19:21,099 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:19:21,100 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-02T09:19:21,100 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:21,100 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:19:21,101 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:19:21,101 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:19:21,102 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:19:21,103 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:19:21,103 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:19:21,103 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T09:19:21,105 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:19:21,107 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T09:19:21,108 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=710236, jitterRate=-0.0968884527683258}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T09:19:21,109 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733131161092Initializing all the Stores at 1733131161093 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131161093Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131161093Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131161093Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131161093Cleaning up temporary data from old regions at 1733131161103 (+10 ms)Region opened successfully at 1733131161109 (+6 ms) 2024-12-02T09:19:21,111 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-02T09:19:21,115 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d48fa3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7c6d666a4939/172.17.0.3:0 2024-12-02T09:19:21,116 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-02T09:19:21,116 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-02T09:19:21,116 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-02T09:19:21,116 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T09:19:21,117 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T09:19:21,118 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-02T09:19:21,118 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-02T09:19:21,121 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-02T09:19:21,122 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-02T09:19:21,153 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-02T09:19:21,154 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-02T09:19:21,155 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-02T09:19:21,165 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-02T09:19:21,166 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-02T09:19:21,167 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-02T09:19:21,173 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-02T09:19:21,175 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-02T09:19:21,181 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-02T09:19:21,184 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-02T09:19:21,190 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-02T09:19:21,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T09:19:21,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38235-0x1009a48aa650001, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T09:19:21,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38235-0x1009a48aa650001, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:21,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:21,199 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7c6d666a4939,42359,1733131160387, sessionid=0x1009a48aa650000, setting cluster-up flag (Was=false) 2024-12-02T09:19:21,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:21,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38235-0x1009a48aa650001, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:21,240 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-02T09:19:21,241 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7c6d666a4939,42359,1733131160387 2024-12-02T09:19:21,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:21,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38235-0x1009a48aa650001, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:21,281 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-02T09:19:21,283 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7c6d666a4939,42359,1733131160387 2024-12-02T09:19:21,284 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-02T09:19:21,286 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T09:19:21,286 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-02T09:19:21,286 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-02T09:19:21,286 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7c6d666a4939,42359,1733131160387 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-02T09:19:21,288 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:19:21,288 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:19:21,288 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:19:21,288 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:19:21,288 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7c6d666a4939:0, corePoolSize=10, maxPoolSize=10 2024-12-02T09:19:21,288 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:19:21,288 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7c6d666a4939:0, corePoolSize=2, maxPoolSize=2 2024-12-02T09:19:21,288 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:19:21,291 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733131191291 2024-12-02T09:19:21,291 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-02T09:19:21,292 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-02T09:19:21,292 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-02T09:19:21,292 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-02T09:19:21,292 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-02T09:19:21,292 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-02T09:19:21,292 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:21,292 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-02T09:19:21,292 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-02T09:19:21,292 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-02T09:19:21,293 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T09:19:21,293 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-02T09:19:21,293 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-02T09:19:21,293 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-02T09:19:21,293 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.large.0-1733131161293,5,FailOnTimeoutGroup] 2024-12-02T09:19:21,293 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.small.0-1733131161293,5,FailOnTimeoutGroup] 2024-12-02T09:19:21,293 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:21,293 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-02T09:19:21,293 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:21,293 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:21,294 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:21,294 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-02T09:19:21,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43343 is added to blk_1073741831_1007 (size=1321) 2024-12-02T09:19:21,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36349 is added to blk_1073741831_1007 (size=1321) 2024-12-02T09:19:21,303 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-02T09:19:21,303 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d 2024-12-02T09:19:21,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36349 is added to blk_1073741832_1008 (size=32) 2024-12-02T09:19:21,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43343 is added to blk_1073741832_1008 (size=32) 2024-12-02T09:19:21,313 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:19:21,314 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T09:19:21,315 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T09:19:21,315 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:21,316 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:19:21,316 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T09:19:21,317 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T09:19:21,317 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:21,318 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:19:21,318 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T09:19:21,319 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T09:19:21,319 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:21,319 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:19:21,319 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T09:19:21,320 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T09:19:21,320 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:21,321 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:19:21,321 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T09:19:21,321 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/hbase/meta/1588230740 2024-12-02T09:19:21,322 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/hbase/meta/1588230740 2024-12-02T09:19:21,323 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T09:19:21,323 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T09:19:21,323 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T09:19:21,324 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T09:19:21,326 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T09:19:21,326 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=863747, jitterRate=0.0983118861913681}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T09:19:21,327 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733131161313Initializing all the Stores at 1733131161314 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131161314Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131161314Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131161314Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131161314Cleaning up temporary data from old regions at 1733131161323 (+9 ms)Region opened successfully at 1733131161327 (+4 ms) 2024-12-02T09:19:21,327 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T09:19:21,327 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T09:19:21,327 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T09:19:21,327 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T09:19:21,327 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T09:19:21,327 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T09:19:21,327 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733131161327Disabling compacts and flushes for region at 1733131161327Disabling writes for close at 1733131161327Writing region close event to WAL at 1733131161327Closed at 1733131161327 2024-12-02T09:19:21,329 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T09:19:21,329 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-02T09:19:21,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-02T09:19:21,330 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T09:19:21,331 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-02T09:19:21,364 INFO [RS:0;7c6d666a4939:38235 {}] regionserver.HRegionServer(746): ClusterId : 1b0443a7-4809-48bd-8e0e-09f20387cb0e 2024-12-02T09:19:21,364 DEBUG [RS:0;7c6d666a4939:38235 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T09:19:21,373 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:21,375 DEBUG [RS:0;7c6d666a4939:38235 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T09:19:21,375 DEBUG [RS:0;7c6d666a4939:38235 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T09:19:21,383 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:21,384 DEBUG [RS:0;7c6d666a4939:38235 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T09:19:21,384 DEBUG [RS:0;7c6d666a4939:38235 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2df99ada, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7c6d666a4939/172.17.0.3:0 2024-12-02T09:19:21,400 DEBUG [RS:0;7c6d666a4939:38235 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7c6d666a4939:38235 2024-12-02T09:19:21,400 INFO [RS:0;7c6d666a4939:38235 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T09:19:21,400 INFO [RS:0;7c6d666a4939:38235 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T09:19:21,400 DEBUG [RS:0;7c6d666a4939:38235 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T09:19:21,401 INFO [RS:0;7c6d666a4939:38235 {}] regionserver.HRegionServer(2659): reportForDuty to master=7c6d666a4939,42359,1733131160387 with port=38235, startcode=1733131160543 2024-12-02T09:19:21,401 DEBUG [RS:0;7c6d666a4939:38235 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T09:19:21,403 INFO [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33591, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T09:19:21,403 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42359 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7c6d666a4939,38235,1733131160543 2024-12-02T09:19:21,403 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42359 {}] master.ServerManager(517): Registering regionserver=7c6d666a4939,38235,1733131160543 2024-12-02T09:19:21,405 DEBUG [RS:0;7c6d666a4939:38235 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d 2024-12-02T09:19:21,405 DEBUG [RS:0;7c6d666a4939:38235 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45419 2024-12-02T09:19:21,405 DEBUG [RS:0;7c6d666a4939:38235 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T09:19:21,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T09:19:21,415 DEBUG [RS:0;7c6d666a4939:38235 {}] zookeeper.ZKUtil(111): regionserver:38235-0x1009a48aa650001, quorum=127.0.0.1:50585, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7c6d666a4939,38235,1733131160543 2024-12-02T09:19:21,416 WARN [RS:0;7c6d666a4939:38235 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T09:19:21,416 INFO [RS:0;7c6d666a4939:38235 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-02T09:19:21,416 DEBUG [RS:0;7c6d666a4939:38235 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543 2024-12-02T09:19:21,416 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7c6d666a4939,38235,1733131160543] 2024-12-02T09:19:21,420 INFO [RS:0;7c6d666a4939:38235 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T09:19:21,422 INFO [RS:0;7c6d666a4939:38235 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T09:19:21,422 INFO [RS:0;7c6d666a4939:38235 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T09:19:21,423 INFO [RS:0;7c6d666a4939:38235 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:21,423 INFO [RS:0;7c6d666a4939:38235 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T09:19:21,424 INFO [RS:0;7c6d666a4939:38235 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T09:19:21,424 INFO [RS:0;7c6d666a4939:38235 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:21,424 DEBUG [RS:0;7c6d666a4939:38235 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:19:21,424 DEBUG [RS:0;7c6d666a4939:38235 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:19:21,424 DEBUG [RS:0;7c6d666a4939:38235 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:19:21,424 DEBUG [RS:0;7c6d666a4939:38235 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:19:21,424 DEBUG [RS:0;7c6d666a4939:38235 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:19:21,424 DEBUG [RS:0;7c6d666a4939:38235 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7c6d666a4939:0, corePoolSize=2, maxPoolSize=2 2024-12-02T09:19:21,424 DEBUG [RS:0;7c6d666a4939:38235 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:19:21,424 DEBUG [RS:0;7c6d666a4939:38235 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:19:21,425 DEBUG [RS:0;7c6d666a4939:38235 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:19:21,425 DEBUG [RS:0;7c6d666a4939:38235 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:19:21,425 DEBUG [RS:0;7c6d666a4939:38235 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:19:21,425 DEBUG [RS:0;7c6d666a4939:38235 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:19:21,425 DEBUG [RS:0;7c6d666a4939:38235 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7c6d666a4939:0, corePoolSize=3, maxPoolSize=3 2024-12-02T09:19:21,425 DEBUG [RS:0;7c6d666a4939:38235 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0, corePoolSize=3, maxPoolSize=3 2024-12-02T09:19:21,425 INFO [RS:0;7c6d666a4939:38235 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:21,425 INFO [RS:0;7c6d666a4939:38235 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:21,426 INFO [RS:0;7c6d666a4939:38235 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:21,426 INFO [RS:0;7c6d666a4939:38235 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:21,426 INFO [RS:0;7c6d666a4939:38235 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:21,426 INFO [RS:0;7c6d666a4939:38235 {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,38235,1733131160543-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T09:19:21,439 INFO [RS:0;7c6d666a4939:38235 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T09:19:21,439 INFO [RS:0;7c6d666a4939:38235 {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,38235,1733131160543-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:21,439 INFO [RS:0;7c6d666a4939:38235 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:21,440 INFO [RS:0;7c6d666a4939:38235 {}] regionserver.Replication(171): 7c6d666a4939,38235,1733131160543 started 2024-12-02T09:19:21,452 INFO [RS:0;7c6d666a4939:38235 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:21,452 INFO [RS:0;7c6d666a4939:38235 {}] regionserver.HRegionServer(1482): Serving as 7c6d666a4939,38235,1733131160543, RpcServer on 7c6d666a4939/172.17.0.3:38235, sessionid=0x1009a48aa650001 2024-12-02T09:19:21,452 DEBUG [RS:0;7c6d666a4939:38235 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T09:19:21,452 DEBUG [RS:0;7c6d666a4939:38235 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7c6d666a4939,38235,1733131160543 2024-12-02T09:19:21,452 DEBUG [RS:0;7c6d666a4939:38235 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7c6d666a4939,38235,1733131160543' 2024-12-02T09:19:21,452 DEBUG [RS:0;7c6d666a4939:38235 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T09:19:21,453 DEBUG [RS:0;7c6d666a4939:38235 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T09:19:21,453 DEBUG [RS:0;7c6d666a4939:38235 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T09:19:21,453 DEBUG [RS:0;7c6d666a4939:38235 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T09:19:21,453 DEBUG [RS:0;7c6d666a4939:38235 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7c6d666a4939,38235,1733131160543 2024-12-02T09:19:21,453 DEBUG [RS:0;7c6d666a4939:38235 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7c6d666a4939,38235,1733131160543' 2024-12-02T09:19:21,453 DEBUG [RS:0;7c6d666a4939:38235 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T09:19:21,453 DEBUG [RS:0;7c6d666a4939:38235 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T09:19:21,454 DEBUG [RS:0;7c6d666a4939:38235 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T09:19:21,454 INFO [RS:0;7c6d666a4939:38235 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T09:19:21,454 INFO [RS:0;7c6d666a4939:38235 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T09:19:21,481 WARN [7c6d666a4939:42359 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-02T09:19:21,559 INFO [RS:0;7c6d666a4939:38235 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c6d666a4939%2C38235%2C1733131160543, suffix=, logDir=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543, archiveDir=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/oldWALs, maxLogs=32 2024-12-02T09:19:21,561 INFO [RS:0;7c6d666a4939:38235 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C38235%2C1733131160543.1733131161561 2024-12-02T09:19:21,570 INFO [RS:0;7c6d666a4939:38235 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131161561 2024-12-02T09:19:21,575 DEBUG [RS:0;7c6d666a4939:38235 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:36145:36145)] 2024-12-02T09:19:21,732 DEBUG [7c6d666a4939:42359 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-02T09:19:21,732 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7c6d666a4939,38235,1733131160543 2024-12-02T09:19:21,734 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7c6d666a4939,38235,1733131160543, state=OPENING 2024-12-02T09:19:21,745 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-02T09:19:21,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:21,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38235-0x1009a48aa650001, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:21,758 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T09:19:21,758 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:19:21,758 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:19:21,758 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7c6d666a4939,38235,1733131160543}] 2024-12-02T09:19:21,915 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T09:19:21,919 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39403, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T09:19:21,925 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-02T09:19:21,925 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-02T09:19:21,927 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c6d666a4939%2C38235%2C1733131160543.meta, suffix=.meta, logDir=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543, archiveDir=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/oldWALs, maxLogs=32 2024-12-02T09:19:21,927 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C38235%2C1733131160543.meta.1733131161927.meta 2024-12-02T09:19:21,933 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.meta.1733131161927.meta 2024-12-02T09:19:21,934 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:36145:36145)] 2024-12-02T09:19:21,935 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-02T09:19:21,936 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-02T09:19:21,936 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-02T09:19:21,936 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-02T09:19:21,936 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-02T09:19:21,936 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:19:21,936 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-02T09:19:21,936 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-02T09:19:21,938 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T09:19:21,939 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T09:19:21,940 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:21,940 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:19:21,940 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T09:19:21,941 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T09:19:21,942 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:21,942 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:19:21,942 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T09:19:21,943 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T09:19:21,943 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:21,944 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:19:21,944 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T09:19:21,944 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T09:19:21,944 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:21,945 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:19:21,945 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T09:19:21,945 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/hbase/meta/1588230740 2024-12-02T09:19:21,946 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/hbase/meta/1588230740 2024-12-02T09:19:21,947 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T09:19:21,947 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T09:19:21,948 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T09:19:21,949 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T09:19:21,950 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=768462, jitterRate=-0.022850513458251953}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T09:19:21,950 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-02T09:19:21,950 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733131161936Writing region info on filesystem at 1733131161937 (+1 ms)Initializing all the Stores at 1733131161938 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131161938Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131161938Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131161938Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131161938Cleaning up temporary data from old regions at 1733131161947 (+9 ms)Running coprocessor post-open hooks at 1733131161950 (+3 ms)Region opened successfully at 1733131161950 2024-12-02T09:19:21,951 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733131161915 2024-12-02T09:19:21,953 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-02T09:19:21,953 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-02T09:19:21,954 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7c6d666a4939,38235,1733131160543 2024-12-02T09:19:21,955 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7c6d666a4939,38235,1733131160543, state=OPEN 2024-12-02T09:19:21,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38235-0x1009a48aa650001, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T09:19:21,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T09:19:21,985 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7c6d666a4939,38235,1733131160543 2024-12-02T09:19:21,986 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:19:21,986 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:19:21,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-02T09:19:21,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7c6d666a4939,38235,1733131160543 in 227 msec 2024-12-02T09:19:21,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-02T09:19:21,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 662 msec 2024-12-02T09:19:21,995 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T09:19:21,995 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-02T09:19:21,997 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T09:19:21,997 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c6d666a4939,38235,1733131160543, seqNum=-1] 2024-12-02T09:19:21,998 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T09:19:21,999 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56899, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T09:19:22,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 718 msec 2024-12-02T09:19:22,004 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733131162004, completionTime=-1 2024-12-02T09:19:22,004 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-02T09:19:22,005 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T09:19:22,006 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-02T09:19:22,006 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733131222006 2024-12-02T09:19:22,006 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733131282006 2024-12-02T09:19:22,006 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec 2024-12-02T09:19:22,006 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,42359,1733131160387-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:22,007 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,42359,1733131160387-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:22,007 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,42359,1733131160387-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:22,007 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7c6d666a4939:42359, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:22,007 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:22,007 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:22,008 DEBUG [master/7c6d666a4939:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-02T09:19:22,010 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.412sec 2024-12-02T09:19:22,010 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-02T09:19:22,010 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-02T09:19:22,010 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-02T09:19:22,010 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-02T09:19:22,010 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-02T09:19:22,010 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,42359,1733131160387-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T09:19:22,010 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,42359,1733131160387-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-02T09:19:22,013 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-02T09:19:22,013 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-02T09:19:22,013 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,42359,1733131160387-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:22,064 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dcfabbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T09:19:22,065 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c6d666a4939,42359,-1 for getting cluster id 2024-12-02T09:19:22,065 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T09:19:22,067 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1b0443a7-4809-48bd-8e0e-09f20387cb0e' 2024-12-02T09:19:22,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T09:19:22,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1b0443a7-4809-48bd-8e0e-09f20387cb0e" 2024-12-02T09:19:22,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dba7c51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T09:19:22,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c6d666a4939,42359,-1] 2024-12-02T09:19:22,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T09:19:22,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:19:22,070 INFO [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T09:19:22,071 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ca8564b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T09:19:22,071 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T09:19:22,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c6d666a4939,38235,1733131160543, seqNum=-1] 2024-12-02T09:19:22,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T09:19:22,074 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T09:19:22,075 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7c6d666a4939,42359,1733131160387 2024-12-02T09:19:22,075 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:19:22,078 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-12-02T09:19:22,078 INFO [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart 2024-12-02T09:19:22,079 INFO [Time-limited test {}] wal.TestLogRolling(323): Replication=2 2024-12-02T09:19:22,079 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T09:19:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is 7c6d666a4939,42359,1733131160387 2024-12-02T09:19:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6a876b0e 2024-12-02T09:19:22,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T09:19:22,082 INFO [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T09:19:22,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-02T09:19:22,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-02T09:19:22,082 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T09:19:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart 2024-12-02T09:19:22,085 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T09:19:22,085 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:22,085 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRollOnPipelineRestart" procId is: 4 2024-12-02T09:19:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T09:19:22,086 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T09:19:22,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43343 is added to blk_1073741835_1011 (size=395) 2024-12-02T09:19:22,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36349 is added to blk_1073741835_1011 (size=395) 2024-12-02T09:19:22,097 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ad8d29959fc2abf63a565d6b9b9b5b2a, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733131162082.ad8d29959fc2abf63a565d6b9b9b5b2a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d 2024-12-02T09:19:22,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43343 is added to blk_1073741836_1012 (size=78) 2024-12-02T09:19:22,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36349 is added to blk_1073741836_1012 (size=78) 2024-12-02T09:19:22,106 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733131162082.ad8d29959fc2abf63a565d6b9b9b5b2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:19:22,107 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing ad8d29959fc2abf63a565d6b9b9b5b2a, disabling compactions & flushes 2024-12-02T09:19:22,107 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733131162082.ad8d29959fc2abf63a565d6b9b9b5b2a. 2024-12-02T09:19:22,107 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733131162082.ad8d29959fc2abf63a565d6b9b9b5b2a. 2024-12-02T09:19:22,107 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733131162082.ad8d29959fc2abf63a565d6b9b9b5b2a. after waiting 0 ms 2024-12-02T09:19:22,107 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733131162082.ad8d29959fc2abf63a565d6b9b9b5b2a. 2024-12-02T09:19:22,107 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733131162082.ad8d29959fc2abf63a565d6b9b9b5b2a. 2024-12-02T09:19:22,107 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for ad8d29959fc2abf63a565d6b9b9b5b2a: Waiting for close lock at 1733131162106Disabling compacts and flushes for region at 1733131162106Disabling writes for close at 1733131162107 (+1 ms)Writing region close event to WAL at 1733131162107Closed at 1733131162107 2024-12-02T09:19:22,108 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T09:19:22,109 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1733131162082.ad8d29959fc2abf63a565d6b9b9b5b2a.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1733131162109"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733131162109"}]},"ts":"1733131162109"} 2024-12-02T09:19:22,111 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T09:19:22,112 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T09:19:22,112 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733131162112"}]},"ts":"1733131162112"} 2024-12-02T09:19:22,114 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta 2024-12-02T09:19:22,114 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=ad8d29959fc2abf63a565d6b9b9b5b2a, ASSIGN}] 2024-12-02T09:19:22,116 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=ad8d29959fc2abf63a565d6b9b9b5b2a, ASSIGN 2024-12-02T09:19:22,117 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=ad8d29959fc2abf63a565d6b9b9b5b2a, ASSIGN; state=OFFLINE, location=7c6d666a4939,38235,1733131160543; forceNewPlan=false, retain=false 2024-12-02T09:19:22,268 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ad8d29959fc2abf63a565d6b9b9b5b2a, regionState=OPENING, regionLocation=7c6d666a4939,38235,1733131160543 2024-12-02T09:19:22,273 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=ad8d29959fc2abf63a565d6b9b9b5b2a, ASSIGN because future has completed 2024-12-02T09:19:22,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad8d29959fc2abf63a565d6b9b9b5b2a, server=7c6d666a4939,38235,1733131160543}] 2024-12-02T09:19:22,374 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:22,384 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:22,435 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1733131162082.ad8d29959fc2abf63a565d6b9b9b5b2a. 2024-12-02T09:19:22,436 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ad8d29959fc2abf63a565d6b9b9b5b2a, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733131162082.ad8d29959fc2abf63a565d6b9b9b5b2a.', STARTKEY => '', ENDKEY => ''} 2024-12-02T09:19:22,436 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart ad8d29959fc2abf63a565d6b9b9b5b2a 2024-12-02T09:19:22,436 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733131162082.ad8d29959fc2abf63a565d6b9b9b5b2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:19:22,436 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ad8d29959fc2abf63a565d6b9b9b5b2a 2024-12-02T09:19:22,437 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ad8d29959fc2abf63a565d6b9b9b5b2a 2024-12-02T09:19:22,439 INFO [StoreOpener-ad8d29959fc2abf63a565d6b9b9b5b2a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region ad8d29959fc2abf63a565d6b9b9b5b2a 2024-12-02T09:19:22,441 INFO [StoreOpener-ad8d29959fc2abf63a565d6b9b9b5b2a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ad8d29959fc2abf63a565d6b9b9b5b2a columnFamilyName info 2024-12-02T09:19:22,441 DEBUG [StoreOpener-ad8d29959fc2abf63a565d6b9b9b5b2a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:22,442 INFO [StoreOpener-ad8d29959fc2abf63a565d6b9b9b5b2a-1 {}] regionserver.HStore(327): Store=ad8d29959fc2abf63a565d6b9b9b5b2a/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:19:22,443 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ad8d29959fc2abf63a565d6b9b9b5b2a 2024-12-02T09:19:22,444 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/default/TestLogRolling-testLogRollOnPipelineRestart/ad8d29959fc2abf63a565d6b9b9b5b2a 2024-12-02T09:19:22,445 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/default/TestLogRolling-testLogRollOnPipelineRestart/ad8d29959fc2abf63a565d6b9b9b5b2a 2024-12-02T09:19:22,446 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ad8d29959fc2abf63a565d6b9b9b5b2a 2024-12-02T09:19:22,446 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ad8d29959fc2abf63a565d6b9b9b5b2a 2024-12-02T09:19:22,449 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ad8d29959fc2abf63a565d6b9b9b5b2a 2024-12-02T09:19:22,451 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/default/TestLogRolling-testLogRollOnPipelineRestart/ad8d29959fc2abf63a565d6b9b9b5b2a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T09:19:22,452 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ad8d29959fc2abf63a565d6b9b9b5b2a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=860538, jitterRate=0.09423083066940308}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T09:19:22,452 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ad8d29959fc2abf63a565d6b9b9b5b2a 2024-12-02T09:19:22,452 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ad8d29959fc2abf63a565d6b9b9b5b2a: Running coprocessor pre-open hook at 1733131162437Writing region info on filesystem at 1733131162437Initializing all the Stores at 1733131162438 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131162438Cleaning up temporary data from old regions at 1733131162446 (+8 ms)Running coprocessor post-open hooks at 1733131162452 (+6 ms)Region opened successfully at 1733131162452 2024-12-02T09:19:22,453 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1733131162082.ad8d29959fc2abf63a565d6b9b9b5b2a., pid=6, masterSystemTime=1733131162430 2024-12-02T09:19:22,455 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1733131162082.ad8d29959fc2abf63a565d6b9b9b5b2a. 2024-12-02T09:19:22,456 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1733131162082.ad8d29959fc2abf63a565d6b9b9b5b2a. 2024-12-02T09:19:22,456 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ad8d29959fc2abf63a565d6b9b9b5b2a, regionState=OPEN, openSeqNum=2, regionLocation=7c6d666a4939,38235,1733131160543 2024-12-02T09:19:22,459 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad8d29959fc2abf63a565d6b9b9b5b2a, server=7c6d666a4939,38235,1733131160543 because future has completed 2024-12-02T09:19:22,463 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-02T09:19:22,463 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ad8d29959fc2abf63a565d6b9b9b5b2a, server=7c6d666a4939,38235,1733131160543 in 186 msec 2024-12-02T09:19:22,466 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-02T09:19:22,466 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=ad8d29959fc2abf63a565d6b9b9b5b2a, ASSIGN in 349 msec 2024-12-02T09:19:22,467 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T09:19:22,467 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733131162467"}]},"ts":"1733131162467"} 2024-12-02T09:19:22,469 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta 2024-12-02T09:19:22,470 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T09:19:22,473 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 388 msec 2024-12-02T09:19:23,376 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:23,386 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:24,377 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:24,387 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:25,338 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-02T09:19:25,351 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:25,351 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:25,351 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:25,352 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:25,352 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:25,353 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:25,355 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:25,355 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:25,355 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:25,358 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:25,378 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:25,388 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:26,379 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:26,389 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:27,380 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:27,389 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:27,420 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-02T09:19:27,421 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart' 2024-12-02T09:19:28,381 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:28,390 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:29,382 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:29,391 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:29,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-02T09:19:29,834 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-02T09:19:29,836 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart 2024-12-02T09:19:29,836 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer 2024-12-02T09:19:29,838 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T09:19:29,838 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-02T09:19:29,838 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-02T09:19:29,838 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-02T09:19:30,382 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:30,392 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:31,384 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:31,393 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:32,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42359 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T09:19:32,142 INFO [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed 2024-12-02T09:19:32,142 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100 2024-12-02T09:19:32,146 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart 2024-12-02T09:19:32,146 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1733131162082.ad8d29959fc2abf63a565d6b9b9b5b2a. 2024-12-02T09:19:32,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1733131162082.ad8d29959fc2abf63a565d6b9b9b5b2a., hostname=7c6d666a4939,38235,1733131160543, seqNum=2] 2024-12-02T09:19:32,385 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:32,394 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:33,386 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:33,395 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:34,154 INFO [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131161561 2024-12-02T09:19:34,155 WARN [ResponseProcessor for block BP-1055095783-172.17.0.3-1733131158781:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1055095783-172.17.0.3-1733131158781:blk_1073741830_1006 java.io.IOException: Bad response ERROR for BP-1055095783-172.17.0.3-1733131158781:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:36349,DS-62cbca48-2a31-4ea9-965d-d8668d707202,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:34,155 WARN [ResponseProcessor for block BP-1055095783-172.17.0.3-1733131158781:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1055095783-172.17.0.3-1733131158781:blk_1073741833_1009 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:34,155 WARN [ResponseProcessor for block BP-1055095783-172.17.0.3-1733131158781:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1055095783-172.17.0.3-1733131158781:blk_1073741834_1010 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:34,156 WARN [DataStreamer for file /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/WALs/7c6d666a4939,42359,1733131160387/7c6d666a4939%2C42359%2C1733131160387.1733131161070 block BP-1055095783-172.17.0.3-1733131158781:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1055095783-172.17.0.3-1733131158781:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43343,DS-a4e5fdd9-9985-482d-a20b-84f74dd61cee,DISK], DatanodeInfoWithStorage[127.0.0.1:36349,DS-62cbca48-2a31-4ea9-965d-d8668d707202,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:36349,DS-62cbca48-2a31-4ea9-965d-d8668d707202,DISK]) is bad. 2024-12-02T09:19:34,156 WARN [DataStreamer for file /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131161561 block BP-1055095783-172.17.0.3-1733131158781:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1055095783-172.17.0.3-1733131158781:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36349,DS-62cbca48-2a31-4ea9-965d-d8668d707202,DISK], DatanodeInfoWithStorage[127.0.0.1:43343,DS-a4e5fdd9-9985-482d-a20b-84f74dd61cee,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36349,DS-62cbca48-2a31-4ea9-965d-d8668d707202,DISK]) is bad. 2024-12-02T09:19:34,156 WARN [DataStreamer for file /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.meta.1733131161927.meta block BP-1055095783-172.17.0.3-1733131158781:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1055095783-172.17.0.3-1733131158781:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36349,DS-62cbca48-2a31-4ea9-965d-d8668d707202,DISK], DatanodeInfoWithStorage[127.0.0.1:43343,DS-a4e5fdd9-9985-482d-a20b-84f74dd61cee,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:36349,DS-62cbca48-2a31-4ea9-965d-d8668d707202,DISK]) is bad. 2024-12-02T09:19:34,156 WARN [PacketResponder: BP-1055095783-172.17.0.3-1733131158781:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:36349] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:19:34,156 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1865235694_22 at /127.0.0.1:46918 [Receiving block BP-1055095783-172.17.0.3-1733131158781:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:43343:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46918 dst: /127.0.0.1:43343 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:19:34,156 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-208090143_22 at /127.0.0.1:38146 [Receiving block BP-1055095783-172.17.0.3-1733131158781:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:36349:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38146 dst: /127.0.0.1:36349 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:19:34,156 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-208090143_22 at /127.0.0.1:38160 [Receiving block BP-1055095783-172.17.0.3-1733131158781:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:36349:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38160 dst: /127.0.0.1:36349 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:19:34,157 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1865235694_22 at /127.0.0.1:38122 [Receiving block BP-1055095783-172.17.0.3-1733131158781:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:36349:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38122 dst: /127.0.0.1:36349 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:19:34,157 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-208090143_22 at /127.0.0.1:46946 [Receiving block BP-1055095783-172.17.0.3-1733131158781:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:43343:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46946 dst: /127.0.0.1:43343 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:19:34,157 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-208090143_22 at /127.0.0.1:46934 [Receiving block BP-1055095783-172.17.0.3-1733131158781:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:43343:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46934 dst: /127.0.0.1:43343 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:19:34,169 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@386ed495{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:19:34,169 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@a4361c3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:19:34,169 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:19:34,169 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@126bd190{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:19:34,170 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@36974255{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/hadoop.log.dir/,STOPPED} 2024-12-02T09:19:34,171 WARN [BP-1055095783-172.17.0.3-1733131158781 heartbeating to localhost/127.0.0.1:45419 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T09:19:34,171 WARN [BP-1055095783-172.17.0.3-1733131158781 heartbeating to localhost/127.0.0.1:45419 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1055095783-172.17.0.3-1733131158781 (Datanode Uuid 0f0f13f0-2e6d-469b-b429-2ecb49eeb705) service to localhost/127.0.0.1:45419 2024-12-02T09:19:34,171 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T09:19:34,171 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T09:19:34,172 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/cluster_efcf4126-ea13-2607-d029-835753d214be/data/data3/current/BP-1055095783-172.17.0.3-1733131158781 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:19:34,172 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/cluster_efcf4126-ea13-2607-d029-835753d214be/data/data4/current/BP-1055095783-172.17.0.3-1733131158781 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:19:34,172 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T09:19:34,183 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:19:34,188 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:19:34,189 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:19:34,189 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:19:34,189 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T09:19:34,189 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e84b526{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:19:34,190 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@592e51be{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:19:34,278 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@9fe4e76{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/java.io.tmpdir/jetty-localhost-45709-hadoop-hdfs-3_4_1-tests_jar-_-any-8850140149303591516/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:19:34,278 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1ede944f{HTTP/1.1, (http/1.1)}{localhost:45709} 2024-12-02T09:19:34,278 INFO [Time-limited test {}] server.Server(415): Started @170759ms 2024-12-02T09:19:34,279 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T09:19:34,296 WARN [ResponseProcessor for block BP-1055095783-172.17.0.3-1733131158781:blk_1073741834_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1055095783-172.17.0.3-1733131158781:blk_1073741834_1013 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:34,296 WARN [ResponseProcessor for block BP-1055095783-172.17.0.3-1733131158781:blk_1073741830_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1055095783-172.17.0.3-1733131158781:blk_1073741830_1015 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:34,296 WARN [ResponseProcessor for block BP-1055095783-172.17.0.3-1733131158781:blk_1073741833_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1055095783-172.17.0.3-1733131158781:blk_1073741833_1014 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:34,296 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-208090143_22 at /127.0.0.1:58894 [Receiving block BP-1055095783-172.17.0.3-1733131158781:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:43343:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58894 dst: /127.0.0.1:43343 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:19:34,296 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1865235694_22 at /127.0.0.1:58922 [Receiving block BP-1055095783-172.17.0.3-1733131158781:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:43343:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58922 dst: /127.0.0.1:43343 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:19:34,297 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-208090143_22 at /127.0.0.1:58910 [Receiving block BP-1055095783-172.17.0.3-1733131158781:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:43343:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58910 dst: /127.0.0.1:43343 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:19:34,298 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@54dbaae8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:19:34,298 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2b27dfb0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:19:34,298 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:19:34,298 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7992aa88{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:19:34,298 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4659fe2a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/hadoop.log.dir/,STOPPED} 2024-12-02T09:19:34,299 WARN [BP-1055095783-172.17.0.3-1733131158781 heartbeating to localhost/127.0.0.1:45419 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T09:19:34,299 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T09:19:34,299 WARN [BP-1055095783-172.17.0.3-1733131158781 heartbeating to localhost/127.0.0.1:45419 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1055095783-172.17.0.3-1733131158781 (Datanode Uuid d63e8879-27d2-488b-8f03-7cddea8478b9) service to localhost/127.0.0.1:45419 2024-12-02T09:19:34,299 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T09:19:34,300 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/cluster_efcf4126-ea13-2607-d029-835753d214be/data/data1/current/BP-1055095783-172.17.0.3-1733131158781 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:19:34,300 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/cluster_efcf4126-ea13-2607-d029-835753d214be/data/data2/current/BP-1055095783-172.17.0.3-1733131158781 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:19:34,300 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T09:19:34,307 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:19:34,309 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:19:34,310 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:19:34,310 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:19:34,310 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T09:19:34,310 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@56744e16{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:19:34,311 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2983f6f4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:19:34,387 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:34,396 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:34,401 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@c9115f6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/java.io.tmpdir/jetty-localhost-45049-hadoop-hdfs-3_4_1-tests_jar-_-any-17901192424522914836/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:19:34,401 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7fbd08cd{HTTP/1.1, (http/1.1)}{localhost:45049} 2024-12-02T09:19:34,401 INFO [Time-limited test {}] server.Server(415): Started @170882ms 2024-12-02T09:19:34,402 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T09:19:34,603 WARN [Thread-1333 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T09:19:34,606 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2bf1866d9c3b6671 with lease ID 0xb843bc2671f9f047: from storage DS-62cbca48-2a31-4ea9-965d-d8668d707202 node DatanodeRegistration(127.0.0.1:46477, datanodeUuid=0f0f13f0-2e6d-469b-b429-2ecb49eeb705, infoPort=46241, infoSecurePort=0, ipcPort=37601, storageInfo=lv=-57;cid=testClusterID;nsid=2056264030;c=1733131158781), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:19:34,606 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2bf1866d9c3b6671 with lease ID 0xb843bc2671f9f047: from storage DS-b0899ce4-10e2-4475-a49b-555d1d2e1329 node DatanodeRegistration(127.0.0.1:46477, datanodeUuid=0f0f13f0-2e6d-469b-b429-2ecb49eeb705, infoPort=46241, infoSecurePort=0, ipcPort=37601, storageInfo=lv=-57;cid=testClusterID;nsid=2056264030;c=1733131158781), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:19:34,771 WARN [Thread-1353 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T09:19:34,773 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa6753cf142a6aa5a with lease ID 0xb843bc2671f9f048: from storage DS-a4e5fdd9-9985-482d-a20b-84f74dd61cee node DatanodeRegistration(127.0.0.1:42103, datanodeUuid=d63e8879-27d2-488b-8f03-7cddea8478b9, infoPort=36059, infoSecurePort=0, ipcPort=44883, storageInfo=lv=-57;cid=testClusterID;nsid=2056264030;c=1733131158781), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-02T09:19:34,773 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa6753cf142a6aa5a with lease ID 0xb843bc2671f9f048: from storage DS-9300c2e6-e373-4ddf-b55d-6512e573e650 node DatanodeRegistration(127.0.0.1:42103, datanodeUuid=d63e8879-27d2-488b-8f03-7cddea8478b9, infoPort=36059, infoSecurePort=0, ipcPort=44883, storageInfo=lv=-57;cid=testClusterID;nsid=2056264030;c=1733131158781), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:19:35,388 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:35,397 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:35,420 INFO [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted 2024-12-02T09:19:35,424 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002 2024-12-02T09:19:35,426 ERROR [FSHLog-0-hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d-prefix:7c6d666a4939,38235,1733131160543 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43343,DS-a4e5fdd9-9985-482d-a20b-84f74dd61cee,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:35,426 WARN [FSHLog-0-hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d-prefix:7c6d666a4939,38235,1733131160543 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43343,DS-a4e5fdd9-9985-482d-a20b-84f74dd61cee,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:35,427 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 7c6d666a4939%2C38235%2C1733131160543:(num 1733131161561) roll requested 2024-12-02T09:19:35,427 INFO [regionserver/7c6d666a4939:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C38235%2C1733131160543.1733131175427 2024-12-02T09:19:35,436 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131161561 newFile=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131175427 2024-12-02T09:19:35,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:35,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:35,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:35,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:35,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:35,437 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131161561 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131175427 2024-12-02T09:19:35,439 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43343,DS-a4e5fdd9-9985-482d-a20b-84f74dd61cee,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:35,439 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43343,DS-a4e5fdd9-9985-482d-a20b-84f74dd61cee,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:35,439 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131161561 2024-12-02T09:19:35,440 WARN [IPC Server handler 0 on default port 45419 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131161561 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1014 2024-12-02T09:19:35,440 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131161561 after 1ms 2024-12-02T09:19:35,443 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36059:36059),(127.0.0.1/127.0.0.1:46241:46241)] 2024-12-02T09:19:35,444 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131161561 is not closed yet, will try archiving it next time 2024-12-02T09:19:36,389 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:36,398 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:37,389 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:37,398 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:37,448 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003 2024-12-02T09:19:37,607 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741833_1014: GenerationStamp not matched, existing replica is blk_1073741833_1009 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-12-02T09:19:38,390 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:38,399 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:39,391 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:39,400 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:39,442 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131161561 after 4002ms 2024-12-02T09:19:39,455 WARN [ResponseProcessor for block BP-1055095783-172.17.0.3-1733131158781:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1055095783-172.17.0.3-1733131158781:blk_1073741837_1016 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:39,455 WARN [DataStreamer for file /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131175427 block BP-1055095783-172.17.0.3-1733131158781:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1055095783-172.17.0.3-1733131158781:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42103,DS-a4e5fdd9-9985-482d-a20b-84f74dd61cee,DISK], DatanodeInfoWithStorage[127.0.0.1:46477,DS-62cbca48-2a31-4ea9-965d-d8668d707202,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42103,DS-a4e5fdd9-9985-482d-a20b-84f74dd61cee,DISK]) is bad. 2024-12-02T09:19:39,456 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-208090143_22 at /127.0.0.1:32900 [Receiving block BP-1055095783-172.17.0.3-1733131158781:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:46477:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:32900 dst: /127.0.0.1:46477 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:19:39,456 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-208090143_22 at /127.0.0.1:41174 [Receiving block BP-1055095783-172.17.0.3-1733131158781:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:42103:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41174 dst: /127.0.0.1:42103 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:19:39,511 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@c9115f6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:19:39,512 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7fbd08cd{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:19:39,512 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:19:39,512 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2983f6f4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:19:39,513 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@56744e16{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/hadoop.log.dir/,STOPPED} 2024-12-02T09:19:39,515 WARN [BP-1055095783-172.17.0.3-1733131158781 heartbeating to localhost/127.0.0.1:45419 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T09:19:39,515 WARN [BP-1055095783-172.17.0.3-1733131158781 heartbeating to localhost/127.0.0.1:45419 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1055095783-172.17.0.3-1733131158781 (Datanode Uuid d63e8879-27d2-488b-8f03-7cddea8478b9) service to localhost/127.0.0.1:45419 2024-12-02T09:19:39,515 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T09:19:39,516 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T09:19:39,517 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/cluster_efcf4126-ea13-2607-d029-835753d214be/data/data1/current/BP-1055095783-172.17.0.3-1733131158781 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:19:39,517 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/cluster_efcf4126-ea13-2607-d029-835753d214be/data/data2/current/BP-1055095783-172.17.0.3-1733131158781 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:19:39,518 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T09:19:39,530 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:19:39,533 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:19:39,534 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:19:39,534 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:19:39,534 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T09:19:39,534 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f78a2f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:19:39,534 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@51065df5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:19:39,624 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1b0a2346{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/java.io.tmpdir/jetty-localhost-40871-hadoop-hdfs-3_4_1-tests_jar-_-any-9216516861897001029/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:19:39,624 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5f3e5a16{HTTP/1.1, (http/1.1)}{localhost:40871} 2024-12-02T09:19:39,624 INFO [Time-limited test {}] server.Server(415): Started @176105ms 2024-12-02T09:19:39,625 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T09:19:39,643 WARN [ResponseProcessor for block BP-1055095783-172.17.0.3-1733131158781:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1055095783-172.17.0.3-1733131158781:blk_1073741837_1018 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:39,644 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-208090143_22 at /127.0.0.1:59110 [Receiving block BP-1055095783-172.17.0.3-1733131158781:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:46477:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59110 dst: /127.0.0.1:46477 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:19:39,646 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@9fe4e76{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:19:39,647 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1ede944f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:19:39,647 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:19:39,647 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@592e51be{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:19:39,647 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e84b526{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/hadoop.log.dir/,STOPPED} 2024-12-02T09:19:39,648 WARN [BP-1055095783-172.17.0.3-1733131158781 heartbeating to localhost/127.0.0.1:45419 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T09:19:39,648 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T09:19:39,648 WARN [BP-1055095783-172.17.0.3-1733131158781 heartbeating to localhost/127.0.0.1:45419 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1055095783-172.17.0.3-1733131158781 (Datanode Uuid 0f0f13f0-2e6d-469b-b429-2ecb49eeb705) service to localhost/127.0.0.1:45419 2024-12-02T09:19:39,648 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T09:19:39,648 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/cluster_efcf4126-ea13-2607-d029-835753d214be/data/data3/current/BP-1055095783-172.17.0.3-1733131158781 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:19:39,649 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/cluster_efcf4126-ea13-2607-d029-835753d214be/data/data4/current/BP-1055095783-172.17.0.3-1733131158781 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:19:39,649 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T09:19:39,661 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:19:39,664 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:19:39,664 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:19:39,664 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:19:39,664 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T09:19:39,665 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6205dc38{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:19:39,665 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29e73e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:19:39,753 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@46f2a49f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/java.io.tmpdir/jetty-localhost-34997-hadoop-hdfs-3_4_1-tests_jar-_-any-5240723168449708606/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:19:39,754 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@462607cc{HTTP/1.1, (http/1.1)}{localhost:34997} 2024-12-02T09:19:39,754 INFO [Time-limited test {}] server.Server(415): Started @176234ms 2024-12-02T09:19:39,755 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T09:19:39,961 WARN [Thread-1407 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T09:19:39,963 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5ed689da6bae53ae with lease ID 0xb843bc2671f9f049: from storage DS-a4e5fdd9-9985-482d-a20b-84f74dd61cee node DatanodeRegistration(127.0.0.1:42121, datanodeUuid=d63e8879-27d2-488b-8f03-7cddea8478b9, infoPort=46201, infoSecurePort=0, ipcPort=34273, storageInfo=lv=-57;cid=testClusterID;nsid=2056264030;c=1733131158781), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-02T09:19:39,963 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5ed689da6bae53ae with lease ID 0xb843bc2671f9f049: from storage DS-9300c2e6-e373-4ddf-b55d-6512e573e650 node DatanodeRegistration(127.0.0.1:42121, datanodeUuid=d63e8879-27d2-488b-8f03-7cddea8478b9, infoPort=46201, infoSecurePort=0, ipcPort=34273, storageInfo=lv=-57;cid=testClusterID;nsid=2056264030;c=1733131158781), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:19:40,091 WARN [Thread-1427 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T09:19:40,093 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfb51045b55792862 with lease ID 0xb843bc2671f9f04a: from storage DS-62cbca48-2a31-4ea9-965d-d8668d707202 node DatanodeRegistration(127.0.0.1:37045, datanodeUuid=0f0f13f0-2e6d-469b-b429-2ecb49eeb705, infoPort=43611, infoSecurePort=0, ipcPort=43367, storageInfo=lv=-57;cid=testClusterID;nsid=2056264030;c=1733131158781), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:19:40,093 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfb51045b55792862 with lease ID 0xb843bc2671f9f04a: from storage DS-b0899ce4-10e2-4475-a49b-555d1d2e1329 node DatanodeRegistration(127.0.0.1:37045, datanodeUuid=0f0f13f0-2e6d-469b-b429-2ecb49eeb705, infoPort=43611, infoSecurePort=0, ipcPort=43367, storageInfo=lv=-57;cid=testClusterID;nsid=2056264030;c=1733131158781), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:19:40,392 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:40,401 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:40,777 INFO [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted 2024-12-02T09:19:40,781 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004 2024-12-02T09:19:40,783 ERROR [FSHLog-0-hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d-prefix:7c6d666a4939,38235,1733131160543 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46477,DS-62cbca48-2a31-4ea9-965d-d8668d707202,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:40,783 WARN [FSHLog-0-hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d-prefix:7c6d666a4939,38235,1733131160543 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46477,DS-62cbca48-2a31-4ea9-965d-d8668d707202,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:40,783 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 7c6d666a4939%2C38235%2C1733131160543:(num 1733131175427) roll requested 2024-12-02T09:19:40,784 INFO [regionserver/7c6d666a4939:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C38235%2C1733131160543.1733131180784 2024-12-02T09:19:40,790 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131175427 newFile=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131180784 2024-12-02T09:19:40,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:40,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:40,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:40,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:40,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:40,790 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131175427 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131180784 2024-12-02T09:19:40,790 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46477,DS-62cbca48-2a31-4ea9-965d-d8668d707202,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:40,791 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46477,DS-62cbca48-2a31-4ea9-965d-d8668d707202,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:40,791 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131175427 2024-12-02T09:19:40,791 WARN [IPC Server handler 4 on default port 45419 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131175427 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018 2024-12-02T09:19:40,791 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43611:43611),(127.0.0.1/127.0.0.1:46201:46201)] 2024-12-02T09:19:40,791 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131175427 is not closed yet, will try archiving it next time 2024-12-02T09:19:40,791 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131175427 after 0ms 2024-12-02T09:19:41,393 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:41,402 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:42,394 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:42,402 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:42,793 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C38235%2C1733131160543.1733131182793 2024-12-02T09:19:42,799 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131180784 newFile=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131182793 2024-12-02T09:19:42,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:42,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:42,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:42,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:42,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:42,800 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131180784 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131182793 2024-12-02T09:19:42,801 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46201:46201),(127.0.0.1/127.0.0.1:43611:43611)] 2024-12-02T09:19:42,801 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131175427 is not closed yet, will try archiving it next time 2024-12-02T09:19:42,801 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131180784 is not closed yet, will try archiving it next time 2024-12-02T09:19:42,801 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131161561 2024-12-02T09:19:42,801 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131161561 2024-12-02T09:19:42,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42121 is added to blk_1073741838_1019 (size=1264) 2024-12-02T09:19:42,803 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131161561 after 1ms 2024-12-02T09:19:42,803 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131161561 2024-12-02T09:19:42,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741838_1019 (size=1264) 2024-12-02T09:19:42,803 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131175427 is not closed yet, will try archiving it next time 2024-12-02T09:19:42,812 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1733131162452/Put/vlen=218/seqid=0] 2024-12-02T09:19:42,812 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1733131172153/Put/vlen=1045/seqid=0] 2024-12-02T09:19:42,812 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131161561 2024-12-02T09:19:42,812 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131175427 2024-12-02T09:19:42,812 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131175427 2024-12-02T09:19:42,813 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131175427 after 1ms 2024-12-02T09:19:42,813 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131175427 2024-12-02T09:19:42,817 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1733131175426/Put/vlen=1045/seqid=0] 2024-12-02T09:19:42,817 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1733131177450/Put/vlen=1045/seqid=0] 2024-12-02T09:19:42,817 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131175427 2024-12-02T09:19:42,817 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131180784 2024-12-02T09:19:42,817 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131180784 2024-12-02T09:19:42,817 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131180784 after 0ms 2024-12-02T09:19:42,817 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131180784 2024-12-02T09:19:42,820 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1733131180783/Put/vlen=1045/seqid=0] 2024-12-02T09:19:42,820 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131182793 2024-12-02T09:19:42,820 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131182793 2024-12-02T09:19:42,821 WARN [IPC Server handler 0 on default port 45419 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131182793 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021 2024-12-02T09:19:42,821 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131182793 after 1ms 2024-12-02T09:19:43,105 WARN [ResponseProcessor for block BP-1055095783-172.17.0.3-1733131158781:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1055095783-172.17.0.3-1733131158781:blk_1073741839_1021 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:43,105 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1865235694_22 at /127.0.0.1:57634 [Receiving block BP-1055095783-172.17.0.3-1733131158781:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:42121:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57634 dst: /127.0.0.1:42121 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:42121 remote=/127.0.0.1:57634]. Total timeout mills is 60000, 59695 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:19:43,106 WARN [DataStreamer for file /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131182793 block BP-1055095783-172.17.0.3-1733131158781:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1055095783-172.17.0.3-1733131158781:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42121,DS-a4e5fdd9-9985-482d-a20b-84f74dd61cee,DISK], DatanodeInfoWithStorage[127.0.0.1:37045,DS-62cbca48-2a31-4ea9-965d-d8668d707202,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42121,DS-a4e5fdd9-9985-482d-a20b-84f74dd61cee,DISK]) is bad. 2024-12-02T09:19:43,105 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1865235694_22 at /127.0.0.1:38768 [Receiving block BP-1055095783-172.17.0.3-1733131158781:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:37045:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38768 dst: /127.0.0.1:37045 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:19:43,107 WARN [DataStreamer for file /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131182793 block BP-1055095783-172.17.0.3-1733131158781:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1055095783-172.17.0.3-1733131158781:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:43,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42121 is added to blk_1073741839_1022 (size=85) 2024-12-02T09:19:43,395 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:43,404 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:43,963 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-12-02T09:19:44,397 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:44,405 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:44,794 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131175427 after 4003ms 2024-12-02T09:19:45,398 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:45,407 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:46,399 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:46,408 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:46,823 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131182793 after 4003ms 2024-12-02T09:19:46,823 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131182793 2024-12-02T09:19:46,832 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131182793 2024-12-02T09:19:46,833 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB 2024-12-02T09:19:46,834 ERROR [FSHLog-0-hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d-prefix:7c6d666a4939,38235,1733131160543.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43343,DS-a4e5fdd9-9985-482d-a20b-84f74dd61cee,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:46,834 WARN [FSHLog-0-hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d-prefix:7c6d666a4939,38235,1733131160543.meta {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43343,DS-a4e5fdd9-9985-482d-a20b-84f74dd61cee,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:46,834 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 7c6d666a4939%2C38235%2C1733131160543.meta:.meta(num 1733131161927) roll requested 2024-12-02T09:19:46,835 INFO [regionserver/7c6d666a4939:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C38235%2C1733131160543.meta.1733131186834.meta 2024-12-02T09:19:46,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:46,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:46,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:46,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:46,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:46,841 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.meta.1733131161927.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.meta.1733131186834.meta 2024-12-02T09:19:46,841 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43343,DS-a4e5fdd9-9985-482d-a20b-84f74dd61cee,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:46,841 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43343,DS-a4e5fdd9-9985-482d-a20b-84f74dd61cee,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:46,841 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.meta.1733131161927.meta 2024-12-02T09:19:46,842 WARN [IPC Server handler 2 on default port 45419 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.meta.1733131161927.meta has not been closed. Lease recovery is in progress. RecoveryId = 1024 for block blk_1073741834_1013 2024-12-02T09:19:46,842 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46201:46201),(127.0.0.1/127.0.0.1:43611:43611)] 2024-12-02T09:19:46,842 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.meta.1733131161927.meta is not closed yet, will try archiving it next time 2024-12-02T09:19:46,842 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.meta.1733131161927.meta after 1ms 2024-12-02T09:19:46,859 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/hbase/meta/1588230740/.tmp/info/4ef6070cc92d48a0aa89d2894d52f710 is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1733131162082.ad8d29959fc2abf63a565d6b9b9b5b2a./info:regioninfo/1733131162456/Put/seqid=0 2024-12-02T09:19:46,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42121 is added to blk_1073741841_1025 (size=7125) 2024-12-02T09:19:46,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741841_1025 (size=7125) 2024-12-02T09:19:46,864 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/hbase/meta/1588230740/.tmp/info/4ef6070cc92d48a0aa89d2894d52f710 2024-12-02T09:19:46,883 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/hbase/meta/1588230740/.tmp/ns/4031d888e6d5465ebe7d29ed9704c480 is 43, key is default/ns:d/1733131161999/Put/seqid=0 2024-12-02T09:19:46,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42121 is added to blk_1073741842_1026 (size=5153) 2024-12-02T09:19:46,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741842_1026 (size=5153) 2024-12-02T09:19:46,889 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/hbase/meta/1588230740/.tmp/ns/4031d888e6d5465ebe7d29ed9704c480 2024-12-02T09:19:46,906 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/hbase/meta/1588230740/.tmp/table/e0e54e103c10457f80b469d0bd5d95a9 is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1733131162467/Put/seqid=0 2024-12-02T09:19:46,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741843_1027 (size=5438) 2024-12-02T09:19:46,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42121 is added to blk_1073741843_1027 (size=5438) 2024-12-02T09:19:46,912 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/hbase/meta/1588230740/.tmp/table/e0e54e103c10457f80b469d0bd5d95a9 2024-12-02T09:19:46,917 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/hbase/meta/1588230740/.tmp/info/4ef6070cc92d48a0aa89d2894d52f710 as hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/hbase/meta/1588230740/info/4ef6070cc92d48a0aa89d2894d52f710 2024-12-02T09:19:46,923 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/hbase/meta/1588230740/info/4ef6070cc92d48a0aa89d2894d52f710, entries=10, sequenceid=11, filesize=7.0 K 2024-12-02T09:19:46,924 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/hbase/meta/1588230740/.tmp/ns/4031d888e6d5465ebe7d29ed9704c480 as hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/hbase/meta/1588230740/ns/4031d888e6d5465ebe7d29ed9704c480 2024-12-02T09:19:46,928 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/hbase/meta/1588230740/ns/4031d888e6d5465ebe7d29ed9704c480, entries=2, sequenceid=11, filesize=5.0 K 2024-12-02T09:19:46,929 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/hbase/meta/1588230740/.tmp/table/e0e54e103c10457f80b469d0bd5d95a9 as hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/hbase/meta/1588230740/table/e0e54e103c10457f80b469d0bd5d95a9 2024-12-02T09:19:46,935 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/hbase/meta/1588230740/table/e0e54e103c10457f80b469d0bd5d95a9, entries=2, sequenceid=11, filesize=5.3 K 2024-12-02T09:19:46,936 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 103ms, sequenceid=11, compaction requested=false 2024-12-02T09:19:46,936 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-12-02T09:19:46,937 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ad8d29959fc2abf63a565d6b9b9b5b2a 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB 2024-12-02T09:19:46,937 ERROR [FSHLog-0-hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d-prefix:7c6d666a4939,38235,1733131160543 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1055095783-172.17.0.3-1733131158781:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:46,937 WARN [FSHLog-0-hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d-prefix:7c6d666a4939,38235,1733131160543 {}] wal.AbstractFSWAL(2174): append entry failed org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1055095783-172.17.0.3-1733131158781:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:46,937 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 7c6d666a4939%2C38235%2C1733131160543:(num 1733131182793) roll requested 2024-12-02T09:19:46,938 INFO [regionserver/7c6d666a4939:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C38235%2C1733131160543.1733131186938 2024-12-02T09:19:46,942 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131182793 newFile=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131186938 2024-12-02T09:19:46,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:46,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:46,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:46,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:46,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:46,942 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131182793 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131186938 2024-12-02T09:19:46,942 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1055095783-172.17.0.3-1733131158781:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:46,943 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1055095783-172.17.0.3-1733131158781:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor103.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:46,943 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131182793 2024-12-02T09:19:46,943 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46201:46201),(127.0.0.1/127.0.0.1:43611:43611)] 2024-12-02T09:19:46,943 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131182793 after 0ms 2024-12-02T09:19:46,943 DEBUG [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131182793 is not closed yet, will try archiving it next time 2024-12-02T09:19:46,944 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.1733131182793 to hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/oldWALs/7c6d666a4939%2C38235%2C1733131160543.1733131182793 2024-12-02T09:19:46,957 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/default/TestLogRolling-testLogRollOnPipelineRestart/ad8d29959fc2abf63a565d6b9b9b5b2a/.tmp/info/330f34c739dd4f3985a1a53c30ac16e7 is 1080, key is row1002/info:/1733131172153/Put/seqid=0 2024-12-02T09:19:46,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42121 is added to blk_1073741845_1029 (size=9270) 2024-12-02T09:19:46,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741845_1029 (size=9270) 2024-12-02T09:19:46,963 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/default/TestLogRolling-testLogRollOnPipelineRestart/ad8d29959fc2abf63a565d6b9b9b5b2a/.tmp/info/330f34c739dd4f3985a1a53c30ac16e7 2024-12-02T09:19:46,969 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/default/TestLogRolling-testLogRollOnPipelineRestart/ad8d29959fc2abf63a565d6b9b9b5b2a/.tmp/info/330f34c739dd4f3985a1a53c30ac16e7 as hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/default/TestLogRolling-testLogRollOnPipelineRestart/ad8d29959fc2abf63a565d6b9b9b5b2a/info/330f34c739dd4f3985a1a53c30ac16e7 2024-12-02T09:19:46,975 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/default/TestLogRolling-testLogRollOnPipelineRestart/ad8d29959fc2abf63a565d6b9b9b5b2a/info/330f34c739dd4f3985a1a53c30ac16e7, entries=4, sequenceid=8, filesize=9.1 K 2024-12-02T09:19:46,976 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for ad8d29959fc2abf63a565d6b9b9b5b2a in 40ms, sequenceid=8, compaction requested=false 2024-12-02T09:19:46,976 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ad8d29959fc2abf63a565d6b9b9b5b2a: 2024-12-02T09:19:46,982 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-02T09:19:46,982 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T09:19:46,983 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T09:19:46,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:19:46,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:19:46,983 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T09:19:46,983 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-02T09:19:46,983 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1793480187, stopped=false 2024-12-02T09:19:46,983 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7c6d666a4939,42359,1733131160387 2024-12-02T09:19:47,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T09:19:47,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38235-0x1009a48aa650001, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T09:19:47,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:47,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38235-0x1009a48aa650001, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:47,028 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T09:19:47,028 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T09:19:47,029 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T09:19:47,029 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:19:47,029 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7c6d666a4939,38235,1733131160543' ***** 2024-12-02T09:19:47,029 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:19:47,029 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-02T09:19:47,030 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38235-0x1009a48aa650001, quorum=127.0.0.1:50585, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:19:47,030 INFO [RS:0;7c6d666a4939:38235 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T09:19:47,030 INFO [RS:0;7c6d666a4939:38235 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-02T09:19:47,030 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T09:19:47,030 INFO [RS:0;7c6d666a4939:38235 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-02T09:19:47,030 INFO [RS:0;7c6d666a4939:38235 {}] regionserver.HRegionServer(3091): Received CLOSE for ad8d29959fc2abf63a565d6b9b9b5b2a 2024-12-02T09:19:47,031 INFO [RS:0;7c6d666a4939:38235 {}] regionserver.HRegionServer(959): stopping server 7c6d666a4939,38235,1733131160543 2024-12-02T09:19:47,031 INFO [RS:0;7c6d666a4939:38235 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T09:19:47,031 INFO [RS:0;7c6d666a4939:38235 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7c6d666a4939:38235. 2024-12-02T09:19:47,031 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ad8d29959fc2abf63a565d6b9b9b5b2a, disabling compactions & flushes 2024-12-02T09:19:47,031 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733131162082.ad8d29959fc2abf63a565d6b9b9b5b2a. 2024-12-02T09:19:47,031 DEBUG [RS:0;7c6d666a4939:38235 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T09:19:47,031 DEBUG [RS:0;7c6d666a4939:38235 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:19:47,031 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733131162082.ad8d29959fc2abf63a565d6b9b9b5b2a. 2024-12-02T09:19:47,032 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733131162082.ad8d29959fc2abf63a565d6b9b9b5b2a. after waiting 0 ms 2024-12-02T09:19:47,032 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733131162082.ad8d29959fc2abf63a565d6b9b9b5b2a. 2024-12-02T09:19:47,032 INFO [RS:0;7c6d666a4939:38235 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T09:19:47,032 INFO [RS:0;7c6d666a4939:38235 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T09:19:47,032 INFO [RS:0;7c6d666a4939:38235 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T09:19:47,032 INFO [RS:0;7c6d666a4939:38235 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-02T09:19:47,032 INFO [RS:0;7c6d666a4939:38235 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-02T09:19:47,032 DEBUG [RS:0;7c6d666a4939:38235 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, ad8d29959fc2abf63a565d6b9b9b5b2a=TestLogRolling-testLogRollOnPipelineRestart,,1733131162082.ad8d29959fc2abf63a565d6b9b9b5b2a.} 2024-12-02T09:19:47,033 DEBUG [RS:0;7c6d666a4939:38235 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, ad8d29959fc2abf63a565d6b9b9b5b2a 2024-12-02T09:19:47,033 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T09:19:47,033 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T09:19:47,033 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T09:19:47,033 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T09:19:47,033 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T09:19:47,038 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/default/TestLogRolling-testLogRollOnPipelineRestart/ad8d29959fc2abf63a565d6b9b9b5b2a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-02T09:19:47,039 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733131162082.ad8d29959fc2abf63a565d6b9b9b5b2a. 2024-12-02T09:19:47,039 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ad8d29959fc2abf63a565d6b9b9b5b2a: Waiting for close lock at 1733131187031Running coprocessor pre-close hooks at 1733131187031Disabling compacts and flushes for region at 1733131187031Disabling writes for close at 1733131187032 (+1 ms)Writing region close event to WAL at 1733131187033 (+1 ms)Running coprocessor post-close hooks at 1733131187039 (+6 ms)Closed at 1733131187039 2024-12-02T09:19:47,039 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733131162082.ad8d29959fc2abf63a565d6b9b9b5b2a. 2024-12-02T09:19:47,039 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-02T09:19:47,040 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T09:19:47,040 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T09:19:47,040 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733131187032Running coprocessor pre-close hooks at 1733131187032Disabling compacts and flushes for region at 1733131187032Disabling writes for close at 1733131187033 (+1 ms)Writing region close event to WAL at 1733131187036 (+3 ms)Running coprocessor post-close hooks at 1733131187040 (+4 ms)Closed at 1733131187040 2024-12-02T09:19:47,040 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-02T09:19:47,233 INFO [RS:0;7c6d666a4939:38235 {}] regionserver.HRegionServer(976): stopping server 7c6d666a4939,38235,1733131160543; all regions closed. 2024-12-02T09:19:47,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:47,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:47,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:47,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:47,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:47,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741840_1023 (size=825) 2024-12-02T09:19:47,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42121 is added to blk_1073741840_1023 (size=825) 2024-12-02T09:19:47,400 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:47,409 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:47,427 INFO [regionserver/7c6d666a4939:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T09:19:47,429 INFO [regionserver/7c6d666a4939:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-02T09:19:47,429 INFO [regionserver/7c6d666a4939:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-02T09:19:48,401 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:48,410 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:49,402 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:49,411 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:49,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T09:19:49,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-02T09:19:49,835 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart 2024-12-02T09:19:50,095 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741834_1013: GenerationStamp not matched, existing replica is blk_1073741834_1010 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-12-02T09:19:50,366 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T09:19:50,404 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:50,412 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:50,844 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.meta.1733131161927.meta after 4003ms 2024-12-02T09:19:50,845 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/WALs/7c6d666a4939,38235,1733131160543/7c6d666a4939%2C38235%2C1733131160543.meta.1733131161927.meta to hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/oldWALs/7c6d666a4939%2C38235%2C1733131160543.meta.1733131161927.meta 2024-12-02T09:19:50,852 DEBUG [RS:0;7c6d666a4939:38235 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/oldWALs 2024-12-02T09:19:50,852 INFO [RS:0;7c6d666a4939:38235 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7c6d666a4939%2C38235%2C1733131160543.meta:.meta(num 1733131186834) 2024-12-02T09:19:50,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:50,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:50,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:50,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:50,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:50,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741844_1028 (size=1162) 2024-12-02T09:19:50,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42121 is added to blk_1073741844_1028 (size=1162) 2024-12-02T09:19:50,861 DEBUG [RS:0;7c6d666a4939:38235 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/oldWALs 2024-12-02T09:19:50,861 INFO [RS:0;7c6d666a4939:38235 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7c6d666a4939%2C38235%2C1733131160543:(num 1733131186938) 2024-12-02T09:19:50,861 DEBUG [RS:0;7c6d666a4939:38235 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:19:50,861 INFO [RS:0;7c6d666a4939:38235 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T09:19:50,861 INFO [RS:0;7c6d666a4939:38235 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T09:19:50,862 INFO [RS:0;7c6d666a4939:38235 {}] hbase.ChoreService(370): Chore service for: regionserver/7c6d666a4939:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-02T09:19:50,862 INFO [RS:0;7c6d666a4939:38235 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T09:19:50,862 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T09:19:50,862 INFO [RS:0;7c6d666a4939:38235 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38235 2024-12-02T09:19:50,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T09:19:50,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38235-0x1009a48aa650001, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7c6d666a4939,38235,1733131160543 2024-12-02T09:19:50,902 INFO [RS:0;7c6d666a4939:38235 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T09:19:50,914 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7c6d666a4939,38235,1733131160543] 2024-12-02T09:19:50,922 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7c6d666a4939,38235,1733131160543 already deleted, retry=false 2024-12-02T09:19:50,922 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7c6d666a4939,38235,1733131160543 expired; onlineServers=0 2024-12-02T09:19:50,922 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7c6d666a4939,42359,1733131160387' ***** 2024-12-02T09:19:50,922 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-02T09:19:50,922 INFO [M:0;7c6d666a4939:42359 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T09:19:50,922 INFO [M:0;7c6d666a4939:42359 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T09:19:50,922 DEBUG [M:0;7c6d666a4939:42359 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-02T09:19:50,922 DEBUG [M:0;7c6d666a4939:42359 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-02T09:19:50,922 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-02T09:19:50,922 DEBUG [master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.large.0-1733131161293 {}] cleaner.HFileCleaner(306): Exit Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.large.0-1733131161293,5,FailOnTimeoutGroup] 2024-12-02T09:19:50,922 DEBUG [master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.small.0-1733131161293 {}] cleaner.HFileCleaner(306): Exit Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.small.0-1733131161293,5,FailOnTimeoutGroup] 2024-12-02T09:19:50,923 INFO [M:0;7c6d666a4939:42359 {}] hbase.ChoreService(370): Chore service for: master/7c6d666a4939:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-02T09:19:50,923 INFO [M:0;7c6d666a4939:42359 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T09:19:50,923 DEBUG [M:0;7c6d666a4939:42359 {}] master.HMaster(1795): Stopping service threads 2024-12-02T09:19:50,923 INFO [M:0;7c6d666a4939:42359 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-02T09:19:50,923 INFO [M:0;7c6d666a4939:42359 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T09:19:50,923 INFO [M:0;7c6d666a4939:42359 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T09:19:50,923 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-02T09:19:50,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-02T09:19:50,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:50,930 DEBUG [M:0;7c6d666a4939:42359 {}] zookeeper.ZKUtil(347): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-02T09:19:50,930 WARN [M:0;7c6d666a4939:42359 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-02T09:19:50,931 INFO [M:0;7c6d666a4939:42359 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/.lastflushedseqids 2024-12-02T09:19:50,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42121 is added to blk_1073741846_1030 (size=111) 2024-12-02T09:19:50,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741846_1030 (size=111) 2024-12-02T09:19:50,939 INFO [M:0;7c6d666a4939:42359 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T09:19:50,939 INFO [M:0;7c6d666a4939:42359 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-02T09:19:50,939 DEBUG [M:0;7c6d666a4939:42359 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T09:19:50,939 INFO [M:0;7c6d666a4939:42359 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:19:50,939 DEBUG [M:0;7c6d666a4939:42359 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:19:50,939 DEBUG [M:0;7c6d666a4939:42359 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T09:19:50,939 DEBUG [M:0;7c6d666a4939:42359 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:19:50,939 INFO [M:0;7c6d666a4939:42359 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.18 KB heapSize=29.16 KB 2024-12-02T09:19:50,940 ERROR [FSHLog-0-hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData-prefix:7c6d666a4939,42359,1733131160387 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43343,DS-a4e5fdd9-9985-482d-a20b-84f74dd61cee,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:50,940 WARN [FSHLog-0-hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData-prefix:7c6d666a4939,42359,1733131160387 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43343,DS-a4e5fdd9-9985-482d-a20b-84f74dd61cee,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:50,940 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog 7c6d666a4939%2C42359%2C1733131160387:(num 1733131161070) roll requested 2024-12-02T09:19:50,940 INFO [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C42359%2C1733131160387.1733131190940 2024-12-02T09:19:50,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:50,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:50,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:50,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:50,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:50,949 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/WALs/7c6d666a4939,42359,1733131160387/7c6d666a4939%2C42359%2C1733131160387.1733131161070 with entries=53, filesize=26.63 KB; new WAL /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/WALs/7c6d666a4939,42359,1733131160387/7c6d666a4939%2C42359%2C1733131160387.1733131190940 2024-12-02T09:19:50,949 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43343,DS-a4e5fdd9-9985-482d-a20b-84f74dd61cee,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:50,949 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:43343,DS-a4e5fdd9-9985-482d-a20b-84f74dd61cee,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-12-02T09:19:50,949 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/WALs/7c6d666a4939,42359,1733131160387/7c6d666a4939%2C42359%2C1733131160387.1733131161070 2024-12-02T09:19:50,950 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43611:43611),(127.0.0.1/127.0.0.1:46201:46201)] 2024-12-02T09:19:50,950 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/WALs/7c6d666a4939,42359,1733131160387/7c6d666a4939%2C42359%2C1733131160387.1733131161070 is not closed yet, will try archiving it next time 2024-12-02T09:19:50,950 WARN [IPC Server handler 2 on default port 45419 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/WALs/7c6d666a4939,42359,1733131160387/7c6d666a4939%2C42359%2C1733131160387.1733131161070 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1015 2024-12-02T09:19:50,950 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/WALs/7c6d666a4939,42359,1733131160387/7c6d666a4939%2C42359%2C1733131160387.1733131161070 after 1ms 2024-12-02T09:19:50,965 DEBUG [M:0;7c6d666a4939:42359 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f15eaba15b9a492089555f1dfa296d89 is 82, key is hbase:meta,,1/info:regioninfo/1733131161954/Put/seqid=0 2024-12-02T09:19:50,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42121 is added to blk_1073741848_1033 (size=5672) 2024-12-02T09:19:50,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741848_1033 (size=5672) 2024-12-02T09:19:50,970 INFO [M:0;7c6d666a4939:42359 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f15eaba15b9a492089555f1dfa296d89 2024-12-02T09:19:50,990 DEBUG [M:0;7c6d666a4939:42359 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9f1ff3243db84f52aef1de7ac0362453 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733131162472/Put/seqid=0 2024-12-02T09:19:50,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42121 is added to blk_1073741849_1034 (size=6119) 2024-12-02T09:19:50,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741849_1034 (size=6119) 2024-12-02T09:19:50,995 INFO [M:0;7c6d666a4939:42359 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.58 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9f1ff3243db84f52aef1de7ac0362453 2024-12-02T09:19:51,013 DEBUG [M:0;7c6d666a4939:42359 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8904781b165c4686a67039a2da095eff is 69, key is 7c6d666a4939,38235,1733131160543/rs:state/1733131161403/Put/seqid=0 2024-12-02T09:19:51,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38235-0x1009a48aa650001, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:19:51,014 INFO [RS:0;7c6d666a4939:38235 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T09:19:51,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38235-0x1009a48aa650001, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:19:51,014 INFO [RS:0;7c6d666a4939:38235 {}] regionserver.HRegionServer(1031): Exiting; stopping=7c6d666a4939,38235,1733131160543; zookeeper connection closed. 2024-12-02T09:19:51,015 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@28cd5d63 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@28cd5d63 2024-12-02T09:19:51,015 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-02T09:19:51,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741850_1035 (size=5156) 2024-12-02T09:19:51,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42121 is added to blk_1073741850_1035 (size=5156) 2024-12-02T09:19:51,018 INFO [M:0;7c6d666a4939:42359 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8904781b165c4686a67039a2da095eff 2024-12-02T09:19:51,036 DEBUG [M:0;7c6d666a4939:42359 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/8620718ceea745a3a919ce13fb7058b5 is 52, key is load_balancer_on/state:d/1733131162077/Put/seqid=0 2024-12-02T09:19:51,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741851_1036 (size=5056) 2024-12-02T09:19:51,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42121 is added to blk_1073741851_1036 (size=5056) 2024-12-02T09:19:51,041 INFO [M:0;7c6d666a4939:42359 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/8620718ceea745a3a919ce13fb7058b5 2024-12-02T09:19:51,048 DEBUG [M:0;7c6d666a4939:42359 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f15eaba15b9a492089555f1dfa296d89 as hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f15eaba15b9a492089555f1dfa296d89 2024-12-02T09:19:51,054 INFO [M:0;7c6d666a4939:42359 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f15eaba15b9a492089555f1dfa296d89, entries=8, sequenceid=56, filesize=5.5 K 2024-12-02T09:19:51,055 DEBUG [M:0;7c6d666a4939:42359 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9f1ff3243db84f52aef1de7ac0362453 as hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9f1ff3243db84f52aef1de7ac0362453 2024-12-02T09:19:51,060 INFO [M:0;7c6d666a4939:42359 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9f1ff3243db84f52aef1de7ac0362453, entries=6, sequenceid=56, filesize=6.0 K 2024-12-02T09:19:51,061 DEBUG [M:0;7c6d666a4939:42359 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8904781b165c4686a67039a2da095eff as hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8904781b165c4686a67039a2da095eff 2024-12-02T09:19:51,066 INFO [M:0;7c6d666a4939:42359 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8904781b165c4686a67039a2da095eff, entries=1, sequenceid=56, filesize=5.0 K 2024-12-02T09:19:51,067 DEBUG [M:0;7c6d666a4939:42359 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/8620718ceea745a3a919ce13fb7058b5 as hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/8620718ceea745a3a919ce13fb7058b5 2024-12-02T09:19:51,072 INFO [M:0;7c6d666a4939:42359 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/8620718ceea745a3a919ce13fb7058b5, entries=1, sequenceid=56, filesize=4.9 K 2024-12-02T09:19:51,073 INFO [M:0;7c6d666a4939:42359 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.18 KB/23738, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 134ms, sequenceid=56, compaction requested=false 2024-12-02T09:19:51,074 INFO [M:0;7c6d666a4939:42359 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:19:51,074 DEBUG [M:0;7c6d666a4939:42359 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733131190939Disabling compacts and flushes for region at 1733131190939Disabling writes for close at 1733131190939Obtaining lock to block concurrent updates at 1733131190939Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733131190939Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23738, getHeapSize=29800, getOffHeapSize=0, getCellsCount=67 at 1733131190940 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733131190950 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733131190950Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733131190964 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733131190964Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733131190975 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733131190989 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733131190989Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733131191001 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733131191012 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733131191012Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733131191023 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733131191036 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733131191036Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@377e45c3: reopening flushed file at 1733131191047 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15a0fbcc: reopening flushed file at 1733131191054 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b5b50b7: reopening flushed file at 1733131191060 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c0023ee: reopening flushed file at 1733131191066 (+6 ms)Finished flush of dataSize ~23.18 KB/23738, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 134ms, sequenceid=56, compaction requested=false at 1733131191073 (+7 ms)Writing region close event to WAL at 1733131191074 (+1 ms)Closed at 1733131191074 2024-12-02T09:19:51,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:51,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:51,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:51,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:51,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:19:51,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42121 is added to blk_1073741847_1031 (size=757) 2024-12-02T09:19:51,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37045 is added to blk_1073741847_1031 (size=757) 2024-12-02T09:19:51,404 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:51,413 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:52,040 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:52,041 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:52,066 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:52,066 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:52,066 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:52,067 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:52,067 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:52,068 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:52,072 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:52,072 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:52,072 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:52,075 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:52,081 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:52,081 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:52,405 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:52,413 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:52,584 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-02T09:19:52,585 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:52,585 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:52,585 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:52,586 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:52,601 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:52,602 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:52,602 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:52,602 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:52,603 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:52,603 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:52,605 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:52,605 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:52,606 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:52,608 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:19:53,095 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741830_1015: GenerationStamp not matched, existing replica is blk_1073741830_1006 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-12-02T09:19:53,406 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:53,414 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:54,407 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:54,415 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:54,951 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/WALs/7c6d666a4939,42359,1733131160387/7c6d666a4939%2C42359%2C1733131160387.1733131161070 after 4002ms 2024-12-02T09:19:54,951 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/WALs/7c6d666a4939,42359,1733131160387/7c6d666a4939%2C42359%2C1733131160387.1733131161070 to hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/oldWALs/7c6d666a4939%2C42359%2C1733131160387.1733131161070 2024-12-02T09:19:54,954 INFO [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/MasterData/oldWALs/7c6d666a4939%2C42359%2C1733131160387.1733131161070 to hdfs://localhost:45419/user/jenkins/test-data/ca323e7c-47cc-ea7a-c999-0a7703d7796d/oldWALs/7c6d666a4939%2C42359%2C1733131160387.1733131161070$masterlocalwal$ 2024-12-02T09:19:54,954 INFO [M:0;7c6d666a4939:42359 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-02T09:19:54,954 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T09:19:54,954 INFO [M:0;7c6d666a4939:42359 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42359 2024-12-02T09:19:54,955 INFO [M:0;7c6d666a4939:42359 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T09:19:55,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:19:55,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42359-0x1009a48aa650000, quorum=127.0.0.1:50585, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:19:55,103 INFO [M:0;7c6d666a4939:42359 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T09:19:55,105 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@46f2a49f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:19:55,107 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@462607cc{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:19:55,107 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:19:55,107 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29e73e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:19:55,108 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6205dc38{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/hadoop.log.dir/,STOPPED} 2024-12-02T09:19:55,113 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T09:19:55,113 WARN [BP-1055095783-172.17.0.3-1733131158781 heartbeating to localhost/127.0.0.1:45419 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T09:19:55,113 WARN [BP-1055095783-172.17.0.3-1733131158781 heartbeating to localhost/127.0.0.1:45419 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1055095783-172.17.0.3-1733131158781 (Datanode Uuid 0f0f13f0-2e6d-469b-b429-2ecb49eeb705) service to localhost/127.0.0.1:45419 2024-12-02T09:19:55,113 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T09:19:55,114 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/cluster_efcf4126-ea13-2607-d029-835753d214be/data/data3/current/BP-1055095783-172.17.0.3-1733131158781 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:19:55,114 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/cluster_efcf4126-ea13-2607-d029-835753d214be/data/data4/current/BP-1055095783-172.17.0.3-1733131158781 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:19:55,115 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T09:19:55,127 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1b0a2346{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:19:55,128 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5f3e5a16{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:19:55,128 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:19:55,128 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@51065df5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:19:55,128 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f78a2f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/hadoop.log.dir/,STOPPED} 2024-12-02T09:19:55,131 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T09:19:55,131 WARN [BP-1055095783-172.17.0.3-1733131158781 heartbeating to localhost/127.0.0.1:45419 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T09:19:55,131 WARN [BP-1055095783-172.17.0.3-1733131158781 heartbeating to localhost/127.0.0.1:45419 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1055095783-172.17.0.3-1733131158781 (Datanode Uuid d63e8879-27d2-488b-8f03-7cddea8478b9) service to localhost/127.0.0.1:45419 2024-12-02T09:19:55,131 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T09:19:55,132 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/cluster_efcf4126-ea13-2607-d029-835753d214be/data/data1/current/BP-1055095783-172.17.0.3-1733131158781 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:19:55,132 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/cluster_efcf4126-ea13-2607-d029-835753d214be/data/data2/current/BP-1055095783-172.17.0.3-1733131158781 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:19:55,132 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T09:19:55,138 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1a48749e{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T09:19:55,139 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@218ef558{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:19:55,139 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:19:55,140 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7091f2a1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:19:55,140 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@341f9f9e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/hadoop.log.dir/,STOPPED} 2024-12-02T09:19:55,152 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-02T09:19:55,181 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-02T09:19:55,191 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=179 (was 154) Potentially hanging thread: nioEventLoopGroup-30-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45419 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-33-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:45419 from jenkins.hfs.4 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-31-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45419 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-33-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-31-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-33-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:45419 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-10-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-32-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-31-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:45419 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:45419 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-10-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-30-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:45419 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-10-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-30-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45419 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-32-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-32-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=229 (was 227) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=677 (was 1009) 2024-12-02T09:19:55,202 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=179, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=229, ProcessCount=11, AvailableMemoryMB=677 2024-12-02T09:19:55,202 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-02T09:19:55,202 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/hadoop.log.dir so I do NOT create it in target/test-data/b1933971-f6de-b51b-8146-8dff4d087525 2024-12-02T09:19:55,202 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a988e3bd-c7c5-2b89-cb1f-ba756c31a142/hadoop.tmp.dir so I do NOT create it in target/test-data/b1933971-f6de-b51b-8146-8dff4d087525 2024-12-02T09:19:55,202 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/cluster_06bbeb12-f01b-3b5f-4167-1dd52b3a1a69, deleteOnExit=true 2024-12-02T09:19:55,202 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-02T09:19:55,203 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/test.cache.data in system properties and HBase conf 2024-12-02T09:19:55,203 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/hadoop.tmp.dir in system properties and HBase conf 2024-12-02T09:19:55,203 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/hadoop.log.dir in system properties and HBase conf 2024-12-02T09:19:55,203 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-02T09:19:55,203 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-02T09:19:55,203 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-02T09:19:55,204 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-02T09:19:55,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-02T09:19:55,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-02T09:19:55,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-02T09:19:55,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T09:19:55,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-02T09:19:55,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-02T09:19:55,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T09:19:55,204 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T09:19:55,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-02T09:19:55,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/nfs.dump.dir in system properties and HBase conf 2024-12-02T09:19:55,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/java.io.tmpdir in system properties and HBase conf 2024-12-02T09:19:55,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T09:19:55,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-02T09:19:55,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-02T09:19:55,223 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-02T09:19:55,409 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:55,416 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:55,457 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:19:55,461 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:19:55,467 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:19:55,467 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:19:55,467 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T09:19:55,468 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:19:55,468 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4ff483aa{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:19:55,469 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@144fb880{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:19:55,575 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@9b25e94{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/java.io.tmpdir/jetty-localhost-38243-hadoop-hdfs-3_4_1-tests_jar-_-any-8848982408429363002/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T09:19:55,576 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3659c62{HTTP/1.1, (http/1.1)}{localhost:38243} 2024-12-02T09:19:55,576 INFO [Time-limited test {}] server.Server(415): Started @192057ms 2024-12-02T09:19:55,594 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-02T09:19:55,763 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:19:55,766 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:19:55,766 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:19:55,766 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:19:55,766 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T09:19:55,767 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@8ac3c66{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:19:55,767 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2f8e2aa3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:19:55,856 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@bff2115{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/java.io.tmpdir/jetty-localhost-42509-hadoop-hdfs-3_4_1-tests_jar-_-any-10386532262714839139/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:19:55,856 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3a2777cc{HTTP/1.1, (http/1.1)}{localhost:42509} 2024-12-02T09:19:55,856 INFO [Time-limited test {}] server.Server(415): Started @192337ms 2024-12-02T09:19:55,857 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T09:19:55,903 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:19:55,908 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:19:55,909 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:19:55,909 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:19:55,909 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T09:19:55,910 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72521dfa{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:19:55,910 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6580d65d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:19:56,007 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@52c57427{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/java.io.tmpdir/jetty-localhost-38567-hadoop-hdfs-3_4_1-tests_jar-_-any-17415847031693488962/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:19:56,007 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2ade404a{HTTP/1.1, (http/1.1)}{localhost:38567} 2024-12-02T09:19:56,007 INFO [Time-limited test {}] server.Server(415): Started @192488ms 2024-12-02T09:19:56,009 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T09:19:56,409 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:56,417 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:56,548 WARN [Thread-1647 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/cluster_06bbeb12-f01b-3b5f-4167-1dd52b3a1a69/data/data1/current/BP-1010008577-172.17.0.3-1733131195238/current, will proceed with Du for space computation calculation, 2024-12-02T09:19:56,548 WARN [Thread-1648 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/cluster_06bbeb12-f01b-3b5f-4167-1dd52b3a1a69/data/data2/current/BP-1010008577-172.17.0.3-1733131195238/current, will proceed with Du for space computation calculation, 2024-12-02T09:19:56,568 WARN [Thread-1611 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T09:19:56,571 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xeddaeded9850ca61 with lease ID 0x37252d83021066df: Processing first storage report for DS-bdd185e7-ab9b-47c0-8579-e043e7f68da4 from datanode DatanodeRegistration(127.0.0.1:36365, datanodeUuid=6ee299e8-cd70-4970-be41-f0ef12e8ad5f, infoPort=40265, infoSecurePort=0, ipcPort=33475, storageInfo=lv=-57;cid=testClusterID;nsid=1935613181;c=1733131195238) 2024-12-02T09:19:56,572 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xeddaeded9850ca61 with lease ID 0x37252d83021066df: from storage DS-bdd185e7-ab9b-47c0-8579-e043e7f68da4 node DatanodeRegistration(127.0.0.1:36365, datanodeUuid=6ee299e8-cd70-4970-be41-f0ef12e8ad5f, infoPort=40265, infoSecurePort=0, ipcPort=33475, storageInfo=lv=-57;cid=testClusterID;nsid=1935613181;c=1733131195238), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:19:56,572 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xeddaeded9850ca61 with lease ID 0x37252d83021066df: Processing first storage report for DS-cdeabc77-4279-4172-a22d-01c87eaccf8a from datanode DatanodeRegistration(127.0.0.1:36365, datanodeUuid=6ee299e8-cd70-4970-be41-f0ef12e8ad5f, infoPort=40265, infoSecurePort=0, ipcPort=33475, storageInfo=lv=-57;cid=testClusterID;nsid=1935613181;c=1733131195238) 2024-12-02T09:19:56,572 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xeddaeded9850ca61 with lease ID 0x37252d83021066df: from storage DS-cdeabc77-4279-4172-a22d-01c87eaccf8a node DatanodeRegistration(127.0.0.1:36365, datanodeUuid=6ee299e8-cd70-4970-be41-f0ef12e8ad5f, infoPort=40265, infoSecurePort=0, ipcPort=33475, storageInfo=lv=-57;cid=testClusterID;nsid=1935613181;c=1733131195238), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:19:56,683 WARN [Thread-1659 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/cluster_06bbeb12-f01b-3b5f-4167-1dd52b3a1a69/data/data4/current/BP-1010008577-172.17.0.3-1733131195238/current, will proceed with Du for space computation calculation, 2024-12-02T09:19:56,683 WARN [Thread-1658 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/cluster_06bbeb12-f01b-3b5f-4167-1dd52b3a1a69/data/data3/current/BP-1010008577-172.17.0.3-1733131195238/current, will proceed with Du for space computation calculation, 2024-12-02T09:19:56,723 WARN [Thread-1634 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T09:19:56,728 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2d4295ed188dbb3b with lease ID 0x37252d83021066e0: Processing first storage report for DS-2e779eab-c9ef-4916-81b3-19a9f4d00600 from datanode DatanodeRegistration(127.0.0.1:41417, datanodeUuid=d6878948-8e77-4140-9d19-62a18a0a7156, infoPort=41289, infoSecurePort=0, ipcPort=33561, storageInfo=lv=-57;cid=testClusterID;nsid=1935613181;c=1733131195238) 2024-12-02T09:19:56,728 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2d4295ed188dbb3b with lease ID 0x37252d83021066e0: from storage DS-2e779eab-c9ef-4916-81b3-19a9f4d00600 node DatanodeRegistration(127.0.0.1:41417, datanodeUuid=d6878948-8e77-4140-9d19-62a18a0a7156, infoPort=41289, infoSecurePort=0, ipcPort=33561, storageInfo=lv=-57;cid=testClusterID;nsid=1935613181;c=1733131195238), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-02T09:19:56,728 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2d4295ed188dbb3b with lease ID 0x37252d83021066e0: Processing first storage report for DS-a74dd290-e5fa-468d-bca3-8cd615827fb1 from datanode DatanodeRegistration(127.0.0.1:41417, datanodeUuid=d6878948-8e77-4140-9d19-62a18a0a7156, infoPort=41289, infoSecurePort=0, ipcPort=33561, storageInfo=lv=-57;cid=testClusterID;nsid=1935613181;c=1733131195238) 2024-12-02T09:19:56,728 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2d4295ed188dbb3b with lease ID 0x37252d83021066e0: from storage DS-a74dd290-e5fa-468d-bca3-8cd615827fb1 node DatanodeRegistration(127.0.0.1:41417, datanodeUuid=d6878948-8e77-4140-9d19-62a18a0a7156, infoPort=41289, infoSecurePort=0, ipcPort=33561, storageInfo=lv=-57;cid=testClusterID;nsid=1935613181;c=1733131195238), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:19:56,740 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525 2024-12-02T09:19:56,756 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/cluster_06bbeb12-f01b-3b5f-4167-1dd52b3a1a69/zookeeper_0, clientPort=63670, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/cluster_06bbeb12-f01b-3b5f-4167-1dd52b3a1a69/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/cluster_06bbeb12-f01b-3b5f-4167-1dd52b3a1a69/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-02T09:19:56,762 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63670 2024-12-02T09:19:56,762 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:19:56,764 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:19:56,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741825_1001 (size=7) 2024-12-02T09:19:56,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741825_1001 (size=7) 2024-12-02T09:19:56,791 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5 with version=8 2024-12-02T09:19:56,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/hbase-staging 2024-12-02T09:19:56,794 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7c6d666a4939:0 server-side Connection retries=45 2024-12-02T09:19:56,794 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:19:56,794 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T09:19:56,794 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T09:19:56,794 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:19:56,794 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T09:19:56,794 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-02T09:19:56,795 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T09:19:56,799 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37959 2024-12-02T09:19:56,801 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37959 connecting to ZooKeeper ensemble=127.0.0.1:63670 2024-12-02T09:19:56,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:379590x0, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T09:19:56,860 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37959-0x1009a4938820000 connected 2024-12-02T09:19:56,947 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:19:56,949 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:19:56,952 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:19:56,952 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5, hbase.cluster.distributed=false 2024-12-02T09:19:56,955 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T09:19:56,963 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37959 2024-12-02T09:19:56,964 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37959 2024-12-02T09:19:56,971 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37959 2024-12-02T09:19:56,975 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37959 2024-12-02T09:19:56,979 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37959 2024-12-02T09:19:57,007 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7c6d666a4939:0 server-side Connection retries=45 2024-12-02T09:19:57,007 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:19:57,007 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T09:19:57,007 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T09:19:57,007 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:19:57,007 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T09:19:57,007 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T09:19:57,007 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T09:19:57,015 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38507 2024-12-02T09:19:57,016 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38507 connecting to ZooKeeper ensemble=127.0.0.1:63670 2024-12-02T09:19:57,018 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:19:57,020 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:19:57,035 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:385070x0, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T09:19:57,036 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38507-0x1009a4938820001 connected 2024-12-02T09:19:57,040 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38507-0x1009a4938820001, quorum=127.0.0.1:63670, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:19:57,040 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T09:19:57,047 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T09:19:57,051 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38507-0x1009a4938820001, quorum=127.0.0.1:63670, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T09:19:57,054 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38507-0x1009a4938820001, quorum=127.0.0.1:63670, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T09:19:57,063 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38507 2024-12-02T09:19:57,063 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38507 2024-12-02T09:19:57,071 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38507 2024-12-02T09:19:57,074 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38507 2024-12-02T09:19:57,074 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38507 2024-12-02T09:19:57,091 DEBUG [M:0;7c6d666a4939:37959 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7c6d666a4939:37959 2024-12-02T09:19:57,095 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7c6d666a4939,37959,1733131196794 2024-12-02T09:19:57,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:19:57,110 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38507-0x1009a4938820001, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:19:57,118 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7c6d666a4939,37959,1733131196794 2024-12-02T09:19:57,131 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38507-0x1009a4938820001, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T09:19:57,131 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38507-0x1009a4938820001, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:57,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:57,140 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-02T09:19:57,147 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7c6d666a4939,37959,1733131196794 from backup master directory 2024-12-02T09:19:57,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7c6d666a4939,37959,1733131196794 2024-12-02T09:19:57,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:19:57,167 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38507-0x1009a4938820001, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:19:57,167 WARN [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T09:19:57,167 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7c6d666a4939,37959,1733131196794 2024-12-02T09:19:57,184 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/hbase.id] with ID: 6814642c-02ca-44e8-81c1-65ee80b2d6b0 2024-12-02T09:19:57,185 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/.tmp/hbase.id 2024-12-02T09:19:57,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741826_1002 (size=42) 2024-12-02T09:19:57,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741826_1002 (size=42) 2024-12-02T09:19:57,410 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:57,417 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:57,603 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/.tmp/hbase.id]:[hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/hbase.id] 2024-12-02T09:19:57,625 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:19:57,626 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-02T09:19:57,627 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-12-02T09:19:57,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:57,638 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38507-0x1009a4938820001, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:57,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741827_1003 (size=196) 2024-12-02T09:19:57,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741827_1003 (size=196) 2024-12-02T09:19:57,660 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T09:19:57,661 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-02T09:19:57,662 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-02T09:19:57,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741828_1004 (size=1189) 2024-12-02T09:19:57,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741828_1004 (size=1189) 2024-12-02T09:19:57,692 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/data/master/store 2024-12-02T09:19:57,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741829_1005 (size=34) 2024-12-02T09:19:57,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741829_1005 (size=34) 2024-12-02T09:19:57,701 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:19:57,701 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T09:19:57,702 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:19:57,702 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:19:57,702 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T09:19:57,702 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:19:57,702 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:19:57,702 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733131197701Disabling compacts and flushes for region at 1733131197701Disabling writes for close at 1733131197702 (+1 ms)Writing region close event to WAL at 1733131197702Closed at 1733131197702 2024-12-02T09:19:57,703 WARN [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/data/master/store/.initializing 2024-12-02T09:19:57,703 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/WALs/7c6d666a4939,37959,1733131196794 2024-12-02T09:19:57,705 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c6d666a4939%2C37959%2C1733131196794, suffix=, logDir=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/WALs/7c6d666a4939,37959,1733131196794, archiveDir=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/oldWALs, maxLogs=10 2024-12-02T09:19:57,706 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C37959%2C1733131196794.1733131197706 2024-12-02T09:19:57,711 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/WALs/7c6d666a4939,37959,1733131196794/7c6d666a4939%2C37959%2C1733131196794.1733131197706 2024-12-02T09:19:57,712 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40265:40265),(127.0.0.1/127.0.0.1:41289:41289)] 2024-12-02T09:19:57,714 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-02T09:19:57,714 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:19:57,714 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:19:57,714 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:19:57,716 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:19:57,717 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-02T09:19:57,717 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:57,718 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:19:57,718 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:19:57,719 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-02T09:19:57,719 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:57,720 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:19:57,720 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:19:57,721 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-02T09:19:57,721 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:57,722 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:19:57,722 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:19:57,723 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-02T09:19:57,723 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:57,724 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:19:57,724 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:19:57,725 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:19:57,725 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:19:57,727 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:19:57,727 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:19:57,728 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T09:19:57,729 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:19:57,730 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T09:19:57,731 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=864406, jitterRate=0.09915018081665039}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T09:19:57,731 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733131197714Initializing all the Stores at 1733131197715 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131197715Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131197715Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131197715Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131197715Cleaning up temporary data from old regions at 1733131197727 (+12 ms)Region opened successfully at 1733131197731 (+4 ms) 2024-12-02T09:19:57,732 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-02T09:19:57,735 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@791bdce1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7c6d666a4939/172.17.0.3:0 2024-12-02T09:19:57,736 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-02T09:19:57,736 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-02T09:19:57,736 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-02T09:19:57,736 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T09:19:57,737 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T09:19:57,738 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-02T09:19:57,738 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-02T09:19:57,740 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-02T09:19:57,741 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-02T09:19:57,747 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-02T09:19:57,747 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-02T09:19:57,748 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-02T09:19:57,755 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-02T09:19:57,755 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-02T09:19:57,757 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-02T09:19:57,763 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-02T09:19:57,764 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-02T09:19:57,772 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-02T09:19:57,774 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-02T09:19:57,780 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-02T09:19:57,788 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38507-0x1009a4938820001, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T09:19:57,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T09:19:57,788 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38507-0x1009a4938820001, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:57,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:57,789 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7c6d666a4939,37959,1733131196794, sessionid=0x1009a4938820000, setting cluster-up flag (Was=false) 2024-12-02T09:19:57,805 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38507-0x1009a4938820001, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:57,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:57,847 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-02T09:19:57,848 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7c6d666a4939,37959,1733131196794 2024-12-02T09:19:57,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:57,864 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38507-0x1009a4938820001, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:57,918 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-02T09:19:57,919 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7c6d666a4939,37959,1733131196794 2024-12-02T09:19:57,920 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-02T09:19:57,922 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T09:19:57,922 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-02T09:19:57,922 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-02T09:19:57,922 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7c6d666a4939,37959,1733131196794 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-02T09:19:57,923 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:19:57,923 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:19:57,923 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:19:57,923 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:19:57,923 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7c6d666a4939:0, corePoolSize=10, maxPoolSize=10 2024-12-02T09:19:57,923 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:19:57,923 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7c6d666a4939:0, corePoolSize=2, maxPoolSize=2 2024-12-02T09:19:57,924 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:19:57,924 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733131227924 2024-12-02T09:19:57,924 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-02T09:19:57,924 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-02T09:19:57,924 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-02T09:19:57,924 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-02T09:19:57,924 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-02T09:19:57,924 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-02T09:19:57,925 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:57,925 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-02T09:19:57,925 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-02T09:19:57,925 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-02T09:19:57,925 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T09:19:57,925 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-02T09:19:57,926 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:57,927 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-02T09:19:57,927 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-02T09:19:57,927 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-02T09:19:57,931 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.large.0-1733131197927,5,FailOnTimeoutGroup] 2024-12-02T09:19:57,933 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.small.0-1733131197931,5,FailOnTimeoutGroup] 2024-12-02T09:19:57,933 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:57,933 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-02T09:19:57,933 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:57,933 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:57,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741831_1007 (size=1321) 2024-12-02T09:19:57,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741831_1007 (size=1321) 2024-12-02T09:19:57,942 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-02T09:19:57,942 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5 2024-12-02T09:19:57,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741832_1008 (size=32) 2024-12-02T09:19:57,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741832_1008 (size=32) 2024-12-02T09:19:57,952 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:19:57,954 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T09:19:57,955 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T09:19:57,955 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:57,955 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:19:57,956 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T09:19:57,957 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T09:19:57,957 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:57,957 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:19:57,957 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T09:19:57,958 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T09:19:57,959 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:57,959 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:19:57,959 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T09:19:57,960 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T09:19:57,960 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:57,961 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:19:57,961 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T09:19:57,962 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/hbase/meta/1588230740 2024-12-02T09:19:57,962 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/hbase/meta/1588230740 2024-12-02T09:19:57,963 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T09:19:57,964 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T09:19:57,964 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T09:19:57,965 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T09:19:57,967 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T09:19:57,967 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=765917, jitterRate=-0.026086866855621338}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T09:19:57,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733131197952Initializing all the Stores at 1733131197953 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131197953Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131197953Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131197953Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131197953Cleaning up temporary data from old regions at 1733131197964 (+11 ms)Region opened successfully at 1733131197968 (+4 ms) 2024-12-02T09:19:57,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T09:19:57,968 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T09:19:57,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T09:19:57,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T09:19:57,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T09:19:57,968 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T09:19:57,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733131197968Disabling compacts and flushes for region at 1733131197968Disabling writes for close at 1733131197968Writing region close event to WAL at 1733131197968Closed at 1733131197968 2024-12-02T09:19:57,970 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T09:19:57,970 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-02T09:19:57,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-02T09:19:57,971 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T09:19:57,973 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-02T09:19:57,978 INFO [RS:0;7c6d666a4939:38507 {}] regionserver.HRegionServer(746): ClusterId : 6814642c-02ca-44e8-81c1-65ee80b2d6b0 2024-12-02T09:19:57,978 DEBUG [RS:0;7c6d666a4939:38507 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T09:19:57,989 DEBUG [RS:0;7c6d666a4939:38507 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T09:19:57,989 DEBUG [RS:0;7c6d666a4939:38507 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T09:19:57,997 DEBUG [RS:0;7c6d666a4939:38507 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T09:19:57,998 DEBUG [RS:0;7c6d666a4939:38507 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4da28fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7c6d666a4939/172.17.0.3:0 2024-12-02T09:19:58,008 DEBUG [RS:0;7c6d666a4939:38507 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7c6d666a4939:38507 2024-12-02T09:19:58,008 INFO [RS:0;7c6d666a4939:38507 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T09:19:58,008 INFO [RS:0;7c6d666a4939:38507 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T09:19:58,008 DEBUG [RS:0;7c6d666a4939:38507 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T09:19:58,008 INFO [RS:0;7c6d666a4939:38507 {}] regionserver.HRegionServer(2659): reportForDuty to master=7c6d666a4939,37959,1733131196794 with port=38507, startcode=1733131197006 2024-12-02T09:19:58,009 DEBUG [RS:0;7c6d666a4939:38507 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T09:19:58,010 INFO [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47125, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T09:19:58,011 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37959 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7c6d666a4939,38507,1733131197006 2024-12-02T09:19:58,011 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37959 {}] master.ServerManager(517): Registering regionserver=7c6d666a4939,38507,1733131197006 2024-12-02T09:19:58,012 DEBUG [RS:0;7c6d666a4939:38507 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5 2024-12-02T09:19:58,012 DEBUG [RS:0;7c6d666a4939:38507 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43733 2024-12-02T09:19:58,012 DEBUG [RS:0;7c6d666a4939:38507 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T09:19:58,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T09:19:58,022 DEBUG [RS:0;7c6d666a4939:38507 {}] zookeeper.ZKUtil(111): regionserver:38507-0x1009a4938820001, quorum=127.0.0.1:63670, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7c6d666a4939,38507,1733131197006 2024-12-02T09:19:58,022 WARN [RS:0;7c6d666a4939:38507 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T09:19:58,022 INFO [RS:0;7c6d666a4939:38507 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-02T09:19:58,022 DEBUG [RS:0;7c6d666a4939:38507 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/WALs/7c6d666a4939,38507,1733131197006 2024-12-02T09:19:58,023 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7c6d666a4939,38507,1733131197006] 2024-12-02T09:19:58,026 INFO [RS:0;7c6d666a4939:38507 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T09:19:58,027 INFO [RS:0;7c6d666a4939:38507 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T09:19:58,027 INFO [RS:0;7c6d666a4939:38507 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T09:19:58,027 INFO [RS:0;7c6d666a4939:38507 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:58,027 INFO [RS:0;7c6d666a4939:38507 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T09:19:58,028 INFO [RS:0;7c6d666a4939:38507 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T09:19:58,028 INFO [RS:0;7c6d666a4939:38507 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:58,028 DEBUG [RS:0;7c6d666a4939:38507 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:19:58,028 DEBUG [RS:0;7c6d666a4939:38507 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:19:58,028 DEBUG [RS:0;7c6d666a4939:38507 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:19:58,028 DEBUG [RS:0;7c6d666a4939:38507 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:19:58,029 DEBUG [RS:0;7c6d666a4939:38507 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:19:58,029 DEBUG [RS:0;7c6d666a4939:38507 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7c6d666a4939:0, corePoolSize=2, maxPoolSize=2 2024-12-02T09:19:58,029 DEBUG [RS:0;7c6d666a4939:38507 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:19:58,029 DEBUG [RS:0;7c6d666a4939:38507 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:19:58,029 DEBUG [RS:0;7c6d666a4939:38507 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:19:58,029 DEBUG [RS:0;7c6d666a4939:38507 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:19:58,029 DEBUG [RS:0;7c6d666a4939:38507 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:19:58,029 DEBUG [RS:0;7c6d666a4939:38507 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:19:58,029 DEBUG [RS:0;7c6d666a4939:38507 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7c6d666a4939:0, corePoolSize=3, maxPoolSize=3 2024-12-02T09:19:58,029 DEBUG [RS:0;7c6d666a4939:38507 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0, corePoolSize=3, maxPoolSize=3 2024-12-02T09:19:58,030 INFO [RS:0;7c6d666a4939:38507 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:58,030 INFO [RS:0;7c6d666a4939:38507 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:58,030 INFO [RS:0;7c6d666a4939:38507 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:58,030 INFO [RS:0;7c6d666a4939:38507 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:58,030 INFO [RS:0;7c6d666a4939:38507 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:58,030 INFO [RS:0;7c6d666a4939:38507 {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,38507,1733131197006-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T09:19:58,045 INFO [RS:0;7c6d666a4939:38507 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T09:19:58,046 INFO [RS:0;7c6d666a4939:38507 {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,38507,1733131197006-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:58,046 INFO [RS:0;7c6d666a4939:38507 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:58,046 INFO [RS:0;7c6d666a4939:38507 {}] regionserver.Replication(171): 7c6d666a4939,38507,1733131197006 started 2024-12-02T09:19:58,059 INFO [RS:0;7c6d666a4939:38507 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:58,059 INFO [RS:0;7c6d666a4939:38507 {}] regionserver.HRegionServer(1482): Serving as 7c6d666a4939,38507,1733131197006, RpcServer on 7c6d666a4939/172.17.0.3:38507, sessionid=0x1009a4938820001 2024-12-02T09:19:58,059 DEBUG [RS:0;7c6d666a4939:38507 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T09:19:58,059 DEBUG [RS:0;7c6d666a4939:38507 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7c6d666a4939,38507,1733131197006 2024-12-02T09:19:58,059 DEBUG [RS:0;7c6d666a4939:38507 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7c6d666a4939,38507,1733131197006' 2024-12-02T09:19:58,059 DEBUG [RS:0;7c6d666a4939:38507 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T09:19:58,060 DEBUG [RS:0;7c6d666a4939:38507 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T09:19:58,061 DEBUG [RS:0;7c6d666a4939:38507 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T09:19:58,061 DEBUG [RS:0;7c6d666a4939:38507 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T09:19:58,061 DEBUG [RS:0;7c6d666a4939:38507 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7c6d666a4939,38507,1733131197006 2024-12-02T09:19:58,061 DEBUG [RS:0;7c6d666a4939:38507 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7c6d666a4939,38507,1733131197006' 2024-12-02T09:19:58,061 DEBUG [RS:0;7c6d666a4939:38507 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T09:19:58,061 DEBUG [RS:0;7c6d666a4939:38507 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T09:19:58,061 DEBUG [RS:0;7c6d666a4939:38507 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T09:19:58,061 INFO [RS:0;7c6d666a4939:38507 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T09:19:58,061 INFO [RS:0;7c6d666a4939:38507 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T09:19:58,123 WARN [7c6d666a4939:37959 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-02T09:19:58,166 INFO [RS:0;7c6d666a4939:38507 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c6d666a4939%2C38507%2C1733131197006, suffix=, logDir=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/WALs/7c6d666a4939,38507,1733131197006, archiveDir=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/oldWALs, maxLogs=32 2024-12-02T09:19:58,167 INFO [RS:0;7c6d666a4939:38507 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C38507%2C1733131197006.1733131198166 2024-12-02T09:19:58,176 INFO [RS:0;7c6d666a4939:38507 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/WALs/7c6d666a4939,38507,1733131197006/7c6d666a4939%2C38507%2C1733131197006.1733131198166 2024-12-02T09:19:58,177 DEBUG [RS:0;7c6d666a4939:38507 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40265:40265),(127.0.0.1/127.0.0.1:41289:41289)] 2024-12-02T09:19:58,373 DEBUG [7c6d666a4939:37959 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-02T09:19:58,374 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7c6d666a4939,38507,1733131197006 2024-12-02T09:19:58,375 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7c6d666a4939,38507,1733131197006, state=OPENING 2024-12-02T09:19:58,388 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-02T09:19:58,397 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38507-0x1009a4938820001, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:58,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:19:58,397 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T09:19:58,397 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:19:58,397 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:19:58,397 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7c6d666a4939,38507,1733131197006}] 2024-12-02T09:19:58,411 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:58,418 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:58,551 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T09:19:58,553 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38565, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T09:19:58,557 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-02T09:19:58,558 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-02T09:19:58,560 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c6d666a4939%2C38507%2C1733131197006.meta, suffix=.meta, logDir=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/WALs/7c6d666a4939,38507,1733131197006, archiveDir=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/oldWALs, maxLogs=32 2024-12-02T09:19:58,561 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C38507%2C1733131197006.meta.1733131198561.meta 2024-12-02T09:19:58,568 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/WALs/7c6d666a4939,38507,1733131197006/7c6d666a4939%2C38507%2C1733131197006.meta.1733131198561.meta 2024-12-02T09:19:58,570 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40265:40265),(127.0.0.1/127.0.0.1:41289:41289)] 2024-12-02T09:19:58,571 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-02T09:19:58,572 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-02T09:19:58,572 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-02T09:19:58,572 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-02T09:19:58,572 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-02T09:19:58,572 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:19:58,572 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-02T09:19:58,572 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-02T09:19:58,575 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T09:19:58,576 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T09:19:58,576 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:58,577 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:19:58,577 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T09:19:58,578 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T09:19:58,578 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:58,579 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:19:58,579 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T09:19:58,580 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T09:19:58,580 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:58,580 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:19:58,580 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T09:19:58,581 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T09:19:58,581 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:58,582 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:19:58,582 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T09:19:58,583 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/hbase/meta/1588230740 2024-12-02T09:19:58,584 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/hbase/meta/1588230740 2024-12-02T09:19:58,585 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T09:19:58,586 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T09:19:58,586 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T09:19:58,587 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T09:19:58,588 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=855303, jitterRate=0.08757480978965759}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T09:19:58,588 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-02T09:19:58,589 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733131198573Writing region info on filesystem at 1733131198573Initializing all the Stores at 1733131198574 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131198574Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131198574Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131198574Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131198574Cleaning up temporary data from old regions at 1733131198586 (+12 ms)Running coprocessor post-open hooks at 1733131198588 (+2 ms)Region opened successfully at 1733131198589 (+1 ms) 2024-12-02T09:19:58,591 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733131198550 2024-12-02T09:19:58,593 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-02T09:19:58,593 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-02T09:19:58,594 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7c6d666a4939,38507,1733131197006 2024-12-02T09:19:58,594 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7c6d666a4939,38507,1733131197006, state=OPEN 2024-12-02T09:19:58,628 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7c6d666a4939,38507,1733131197006 2024-12-02T09:19:58,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T09:19:58,628 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38507-0x1009a4938820001, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T09:19:58,628 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:19:58,628 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:19:58,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-02T09:19:58,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7c6d666a4939,38507,1733131197006 in 231 msec 2024-12-02T09:19:58,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-02T09:19:58,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 661 msec 2024-12-02T09:19:58,635 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T09:19:58,635 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-02T09:19:58,636 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T09:19:58,637 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c6d666a4939,38507,1733131197006, seqNum=-1] 2024-12-02T09:19:58,637 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T09:19:58,640 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33283, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T09:19:58,648 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 725 msec 2024-12-02T09:19:58,648 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733131198648, completionTime=-1 2024-12-02T09:19:58,648 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-02T09:19:58,648 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T09:19:58,651 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-02T09:19:58,651 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733131258651 2024-12-02T09:19:58,651 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733131318651 2024-12-02T09:19:58,651 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-12-02T09:19:58,651 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,37959,1733131196794-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:58,651 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,37959,1733131196794-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:58,651 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,37959,1733131196794-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:58,652 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7c6d666a4939:37959, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:58,652 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:58,652 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:58,654 DEBUG [master/7c6d666a4939:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-02T09:19:58,656 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.488sec 2024-12-02T09:19:58,656 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-02T09:19:58,656 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-02T09:19:58,656 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-02T09:19:58,656 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-02T09:19:58,656 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-02T09:19:58,656 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,37959,1733131196794-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T09:19:58,656 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,37959,1733131196794-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-02T09:19:58,659 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-02T09:19:58,659 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-02T09:19:58,659 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,37959,1733131196794-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:19:58,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dcbd726, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T09:19:58,678 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c6d666a4939,37959,-1 for getting cluster id 2024-12-02T09:19:58,678 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T09:19:58,680 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6814642c-02ca-44e8-81c1-65ee80b2d6b0' 2024-12-02T09:19:58,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T09:19:58,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6814642c-02ca-44e8-81c1-65ee80b2d6b0" 2024-12-02T09:19:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7756c79e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T09:19:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c6d666a4939,37959,-1] 2024-12-02T09:19:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T09:19:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:19:58,682 INFO [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T09:19:58,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d9a0032, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T09:19:58,684 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T09:19:58,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c6d666a4939,38507,1733131197006, seqNum=-1] 2024-12-02T09:19:58,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T09:19:58,686 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T09:19:58,688 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7c6d666a4939,37959,1733131196794 2024-12-02T09:19:58,688 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:19:58,690 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-12-02T09:19:58,691 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T09:19:58,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is 7c6d666a4939,37959,1733131196794 2024-12-02T09:19:58,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2f022144 2024-12-02T09:19:58,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T09:19:58,693 INFO [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T09:19:58,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37959 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-02T09:19:58,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37959 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-02T09:19:58,694 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37959 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T09:19:58,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37959 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-02T09:19:58,696 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T09:19:58,696 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:58,696 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37959 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling" procId is: 4 2024-12-02T09:19:58,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T09:19:58,697 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T09:19:58,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741835_1011 (size=405) 2024-12-02T09:19:58,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741835_1011 (size=405) 2024-12-02T09:19:58,706 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ed2a52e34f710bc9c88f30746da808e9, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5 2024-12-02T09:19:58,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741836_1012 (size=88) 2024-12-02T09:19:58,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741836_1012 (size=88) 2024-12-02T09:19:58,717 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:19:58,717 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing ed2a52e34f710bc9c88f30746da808e9, disabling compactions & flushes 2024-12-02T09:19:58,717 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9. 2024-12-02T09:19:58,717 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9. 2024-12-02T09:19:58,717 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9. after waiting 0 ms 2024-12-02T09:19:58,717 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9. 2024-12-02T09:19:58,717 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9. 2024-12-02T09:19:58,717 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for ed2a52e34f710bc9c88f30746da808e9: Waiting for close lock at 1733131198717Disabling compacts and flushes for region at 1733131198717Disabling writes for close at 1733131198717Writing region close event to WAL at 1733131198717Closed at 1733131198717 2024-12-02T09:19:58,718 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T09:19:58,719 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1733131198718"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733131198718"}]},"ts":"1733131198718"} 2024-12-02T09:19:58,721 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T09:19:58,722 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T09:19:58,722 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733131198722"}]},"ts":"1733131198722"} 2024-12-02T09:19:58,724 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta 2024-12-02T09:19:58,724 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=ed2a52e34f710bc9c88f30746da808e9, ASSIGN}] 2024-12-02T09:19:58,725 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=ed2a52e34f710bc9c88f30746da808e9, ASSIGN 2024-12-02T09:19:58,726 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=ed2a52e34f710bc9c88f30746da808e9, ASSIGN; state=OFFLINE, location=7c6d666a4939,38507,1733131197006; forceNewPlan=false, retain=false 2024-12-02T09:19:58,877 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ed2a52e34f710bc9c88f30746da808e9, regionState=OPENING, regionLocation=7c6d666a4939,38507,1733131197006 2024-12-02T09:19:58,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=ed2a52e34f710bc9c88f30746da808e9, ASSIGN because future has completed 2024-12-02T09:19:58,880 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed2a52e34f710bc9c88f30746da808e9, server=7c6d666a4939,38507,1733131197006}] 2024-12-02T09:19:59,037 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9. 2024-12-02T09:19:59,037 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ed2a52e34f710bc9c88f30746da808e9, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9.', STARTKEY => '', ENDKEY => ''} 2024-12-02T09:19:59,037 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling ed2a52e34f710bc9c88f30746da808e9 2024-12-02T09:19:59,038 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:19:59,038 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ed2a52e34f710bc9c88f30746da808e9 2024-12-02T09:19:59,038 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ed2a52e34f710bc9c88f30746da808e9 2024-12-02T09:19:59,039 INFO [StoreOpener-ed2a52e34f710bc9c88f30746da808e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region ed2a52e34f710bc9c88f30746da808e9 2024-12-02T09:19:59,041 INFO [StoreOpener-ed2a52e34f710bc9c88f30746da808e9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed2a52e34f710bc9c88f30746da808e9 columnFamilyName info 2024-12-02T09:19:59,041 DEBUG [StoreOpener-ed2a52e34f710bc9c88f30746da808e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:19:59,042 INFO [StoreOpener-ed2a52e34f710bc9c88f30746da808e9-1 {}] regionserver.HStore(327): Store=ed2a52e34f710bc9c88f30746da808e9/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:19:59,042 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ed2a52e34f710bc9c88f30746da808e9 2024-12-02T09:19:59,043 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9 2024-12-02T09:19:59,043 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9 2024-12-02T09:19:59,043 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ed2a52e34f710bc9c88f30746da808e9 2024-12-02T09:19:59,044 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ed2a52e34f710bc9c88f30746da808e9 2024-12-02T09:19:59,045 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ed2a52e34f710bc9c88f30746da808e9 2024-12-02T09:19:59,048 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T09:19:59,048 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ed2a52e34f710bc9c88f30746da808e9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=737453, jitterRate=-0.06228122115135193}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T09:19:59,049 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ed2a52e34f710bc9c88f30746da808e9 2024-12-02T09:19:59,049 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ed2a52e34f710bc9c88f30746da808e9: Running coprocessor pre-open hook at 1733131199038Writing region info on filesystem at 1733131199038Initializing all the Stores at 1733131199039 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131199039Cleaning up temporary data from old regions at 1733131199044 (+5 ms)Running coprocessor post-open hooks at 1733131199049 (+5 ms)Region opened successfully at 1733131199049 2024-12-02T09:19:59,050 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9., pid=6, masterSystemTime=1733131199033 2024-12-02T09:19:59,053 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9. 2024-12-02T09:19:59,053 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9. 2024-12-02T09:19:59,054 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ed2a52e34f710bc9c88f30746da808e9, regionState=OPEN, openSeqNum=2, regionLocation=7c6d666a4939,38507,1733131197006 2024-12-02T09:19:59,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed2a52e34f710bc9c88f30746da808e9, server=7c6d666a4939,38507,1733131197006 because future has completed 2024-12-02T09:19:59,060 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-02T09:19:59,061 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ed2a52e34f710bc9c88f30746da808e9, server=7c6d666a4939,38507,1733131197006 in 178 msec 2024-12-02T09:19:59,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-02T09:19:59,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=ed2a52e34f710bc9c88f30746da808e9, ASSIGN in 337 msec 2024-12-02T09:19:59,065 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T09:19:59,065 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733131199065"}]},"ts":"1733131199065"} 2024-12-02T09:19:59,067 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta 2024-12-02T09:19:59,068 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T09:19:59,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 374 msec 2024-12-02T09:19:59,412 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:59,418 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:19:59,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-02T09:19:59,834 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-02T09:19:59,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T09:19:59,835 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-02T09:19:59,835 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-02T09:19:59,835 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer 2024-12-02T09:20:00,412 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:00,419 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:01,413 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:01,420 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:02,414 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:02,421 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:03,415 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:03,422 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:03,573 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:03,573 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:03,574 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:03,574 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:03,574 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:03,575 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:03,590 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:03,590 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:03,590 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:03,590 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:03,591 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:03,591 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:03,593 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:03,593 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:03,593 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:03,594 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:04,099 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-02T09:20:04,102 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:04,103 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:04,103 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:04,103 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:04,104 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:04,104 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:04,120 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:04,120 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:04,120 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:04,121 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:04,121 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:04,121 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:04,124 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:04,124 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:04,124 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:04,126 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:04,131 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-02T09:20:04,132 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling' 2024-12-02T09:20:04,416 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:04,423 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:05,417 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:05,424 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:06,418 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:06,424 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:07,419 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:07,425 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:08,420 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:08,426 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:08,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T09:20:08,782 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-12-02T09:20:08,782 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100 2024-12-02T09:20:08,785 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-02T09:20:08,785 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9. 2024-12-02T09:20:08,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9., hostname=7c6d666a4939,38507,1733131197006, seqNum=2] 2024-12-02T09:20:08,794 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-02T09:20:08,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37959 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-02T09:20:08,800 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-12-02T09:20:08,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T09:20:08,801 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T09:20:08,803 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T09:20:08,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38507 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-02T09:20:08,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9. 2024-12-02T09:20:08,964 INFO [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing ed2a52e34f710bc9c88f30746da808e9 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-02T09:20:08,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/.tmp/info/4cbc915f1ac14d0aa83318051c2b5295 is 1080, key is row0001/info:/1733131208789/Put/seqid=0 2024-12-02T09:20:08,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741837_1013 (size=6033) 2024-12-02T09:20:08,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741837_1013 (size=6033) 2024-12-02T09:20:09,384 INFO [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/.tmp/info/4cbc915f1ac14d0aa83318051c2b5295 2024-12-02T09:20:09,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/.tmp/info/4cbc915f1ac14d0aa83318051c2b5295 as hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/info/4cbc915f1ac14d0aa83318051c2b5295 2024-12-02T09:20:09,397 INFO [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/info/4cbc915f1ac14d0aa83318051c2b5295, entries=1, sequenceid=5, filesize=5.9 K 2024-12-02T09:20:09,398 INFO [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for ed2a52e34f710bc9c88f30746da808e9 in 434ms, sequenceid=5, compaction requested=false 2024-12-02T09:20:09,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for ed2a52e34f710bc9c88f30746da808e9: 2024-12-02T09:20:09,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9. 2024-12-02T09:20:09,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-02T09:20:09,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37959 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-02T09:20:09,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-02T09:20:09,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 601 msec 2024-12-02T09:20:09,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 612 msec 2024-12-02T09:20:09,421 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:09,427 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:10,422 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:10,427 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:11,422 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:11,428 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:12,423 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:12,428 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:13,423 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:13,429 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:14,424 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:14,429 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:15,425 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:15,430 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:16,425 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:16,430 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:17,426 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:17,431 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:18,426 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:18,431 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:18,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-02T09:20:18,852 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-12-02T09:20:18,855 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-02T09:20:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37959 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-02T09:20:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T09:20:18,857 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-12-02T09:20:18,858 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T09:20:18,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T09:20:19,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38507 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-02T09:20:19,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9. 2024-12-02T09:20:19,012 INFO [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing ed2a52e34f710bc9c88f30746da808e9 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-02T09:20:19,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/.tmp/info/bc779e7cf3b34b85bd8c83941a9c36e0 is 1080, key is row0002/info:/1733131218853/Put/seqid=0 2024-12-02T09:20:19,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741838_1014 (size=6033) 2024-12-02T09:20:19,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741838_1014 (size=6033) 2024-12-02T09:20:19,022 INFO [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/.tmp/info/bc779e7cf3b34b85bd8c83941a9c36e0 2024-12-02T09:20:19,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/.tmp/info/bc779e7cf3b34b85bd8c83941a9c36e0 as hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/info/bc779e7cf3b34b85bd8c83941a9c36e0 2024-12-02T09:20:19,033 INFO [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/info/bc779e7cf3b34b85bd8c83941a9c36e0, entries=1, sequenceid=9, filesize=5.9 K 2024-12-02T09:20:19,034 INFO [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for ed2a52e34f710bc9c88f30746da808e9 in 22ms, sequenceid=9, compaction requested=false 2024-12-02T09:20:19,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for ed2a52e34f710bc9c88f30746da808e9: 2024-12-02T09:20:19,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9. 2024-12-02T09:20:19,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-02T09:20:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37959 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-02T09:20:19,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-02T09:20:19,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 179 msec 2024-12-02T09:20:19,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 186 msec 2024-12-02T09:20:19,427 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:19,427 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 after 68064ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor192.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:20:19,432 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:19,432 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta after 68055ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor192.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T09:20:20,428 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:20,432 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:21,428 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:21,433 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:22,429 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:22,433 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:23,429 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:23,433 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:24,430 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:24,434 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:25,431 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:25,434 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:26,431 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:26,434 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:26,739 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T09:20:27,432 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:27,435 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:28,433 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:28,435 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:28,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T09:20:28,952 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-12-02T09:20:28,955 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C38507%2C1733131197006.1733131228954 2024-12-02T09:20:28,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:28,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:28,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:28,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:28,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:28,961 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/WALs/7c6d666a4939,38507,1733131197006/7c6d666a4939%2C38507%2C1733131197006.1733131198166 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/WALs/7c6d666a4939,38507,1733131197006/7c6d666a4939%2C38507%2C1733131197006.1733131228954 2024-12-02T09:20:28,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741833_1009 (size=5546) 2024-12-02T09:20:28,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741833_1009 (size=5546) 2024-12-02T09:20:28,963 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40265:40265),(127.0.0.1/127.0.0.1:41289:41289)] 2024-12-02T09:20:28,963 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/WALs/7c6d666a4939,38507,1733131197006/7c6d666a4939%2C38507%2C1733131197006.1733131198166 is not closed yet, will try archiving it next time 2024-12-02T09:20:28,964 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-02T09:20:28,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37959 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-02T09:20:28,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T09:20:28,967 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-12-02T09:20:28,968 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T09:20:28,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T09:20:29,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38507 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-02T09:20:29,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9. 2024-12-02T09:20:29,121 INFO [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing ed2a52e34f710bc9c88f30746da808e9 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-02T09:20:29,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/.tmp/info/d751e309654d4fd1ac6d599c02ca2dfd is 1080, key is row0003/info:/1733131228953/Put/seqid=0 2024-12-02T09:20:29,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741840_1016 (size=6033) 2024-12-02T09:20:29,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741840_1016 (size=6033) 2024-12-02T09:20:29,131 INFO [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/.tmp/info/d751e309654d4fd1ac6d599c02ca2dfd 2024-12-02T09:20:29,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/.tmp/info/d751e309654d4fd1ac6d599c02ca2dfd as hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/info/d751e309654d4fd1ac6d599c02ca2dfd 2024-12-02T09:20:29,146 INFO [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/info/d751e309654d4fd1ac6d599c02ca2dfd, entries=1, sequenceid=13, filesize=5.9 K 2024-12-02T09:20:29,147 INFO [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for ed2a52e34f710bc9c88f30746da808e9 in 26ms, sequenceid=13, compaction requested=true 2024-12-02T09:20:29,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for ed2a52e34f710bc9c88f30746da808e9: 2024-12-02T09:20:29,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9. 2024-12-02T09:20:29,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-02T09:20:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37959 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-02T09:20:29,152 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-02T09:20:29,152 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 181 msec 2024-12-02T09:20:29,155 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 189 msec 2024-12-02T09:20:29,433 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:29,436 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:30,434 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:30,436 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:31,435 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:31,437 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:32,436 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:32,438 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:33,437 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:33,438 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:34,437 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:34,439 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:35,438 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:35,439 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:36,438 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:36,439 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:37,439 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:37,440 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:38,440 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:38,441 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:38,660 INFO [master/7c6d666a4939:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-02T09:20:38,660 INFO [master/7c6d666a4939:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-02T09:20:39,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T09:20:39,002 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-12-02T09:20:39,002 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T09:20:39,003 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T09:20:39,003 DEBUG [Time-limited test {}] regionserver.HStore(1541): ed2a52e34f710bc9c88f30746da808e9/info is initiating minor compaction (all files) 2024-12-02T09:20:39,003 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T09:20:39,003 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:20:39,003 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of ed2a52e34f710bc9c88f30746da808e9/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9. 2024-12-02T09:20:39,004 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/info/4cbc915f1ac14d0aa83318051c2b5295, hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/info/bc779e7cf3b34b85bd8c83941a9c36e0, hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/info/d751e309654d4fd1ac6d599c02ca2dfd] into tmpdir=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/.tmp, totalSize=17.7 K 2024-12-02T09:20:39,004 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 4cbc915f1ac14d0aa83318051c2b5295, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733131208789 2024-12-02T09:20:39,004 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting bc779e7cf3b34b85bd8c83941a9c36e0, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1733131218853 2024-12-02T09:20:39,005 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting d751e309654d4fd1ac6d599c02ca2dfd, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733131228953 2024-12-02T09:20:39,016 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): ed2a52e34f710bc9c88f30746da808e9#info#compaction#45 average throughput is 3.08 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T09:20:39,016 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/.tmp/info/979593cc4e5d47bea886caf59a78fbee is 1080, key is row0001/info:/1733131208789/Put/seqid=0 2024-12-02T09:20:39,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741841_1017 (size=8296) 2024-12-02T09:20:39,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741841_1017 (size=8296) 2024-12-02T09:20:39,027 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/.tmp/info/979593cc4e5d47bea886caf59a78fbee as hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/info/979593cc4e5d47bea886caf59a78fbee 2024-12-02T09:20:39,034 INFO [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ed2a52e34f710bc9c88f30746da808e9/info of ed2a52e34f710bc9c88f30746da808e9 into 979593cc4e5d47bea886caf59a78fbee(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T09:20:39,034 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for ed2a52e34f710bc9c88f30746da808e9: 2024-12-02T09:20:39,037 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C38507%2C1733131197006.1733131239037 2024-12-02T09:20:39,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:39,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:39,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:39,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:39,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:39,049 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/WALs/7c6d666a4939,38507,1733131197006/7c6d666a4939%2C38507%2C1733131197006.1733131228954 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/WALs/7c6d666a4939,38507,1733131197006/7c6d666a4939%2C38507%2C1733131197006.1733131239037 2024-12-02T09:20:39,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741839_1015 (size=2520) 2024-12-02T09:20:39,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741839_1015 (size=2520) 2024-12-02T09:20:39,055 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/WALs/7c6d666a4939,38507,1733131197006/7c6d666a4939%2C38507%2C1733131197006.1733131198166 to hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/oldWALs/7c6d666a4939%2C38507%2C1733131197006.1733131198166 2024-12-02T09:20:39,061 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40265:40265),(127.0.0.1/127.0.0.1:41289:41289)] 2024-12-02T09:20:39,062 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37959 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-02T09:20:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37959 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-02T09:20:39,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T09:20:39,064 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-12-02T09:20:39,065 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-02T09:20:39,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-02T09:20:39,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38507 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-02T09:20:39,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9. 2024-12-02T09:20:39,218 INFO [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing ed2a52e34f710bc9c88f30746da808e9 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-02T09:20:39,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/.tmp/info/602f1b093eb24645a408dae48b401417 is 1080, key is row0000/info:/1733131239035/Put/seqid=0 2024-12-02T09:20:39,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741843_1019 (size=6033) 2024-12-02T09:20:39,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741843_1019 (size=6033) 2024-12-02T09:20:39,235 INFO [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/.tmp/info/602f1b093eb24645a408dae48b401417 2024-12-02T09:20:39,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/.tmp/info/602f1b093eb24645a408dae48b401417 as hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/info/602f1b093eb24645a408dae48b401417 2024-12-02T09:20:39,246 INFO [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/info/602f1b093eb24645a408dae48b401417, entries=1, sequenceid=18, filesize=5.9 K 2024-12-02T09:20:39,247 INFO [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for ed2a52e34f710bc9c88f30746da808e9 in 29ms, sequenceid=18, compaction requested=false 2024-12-02T09:20:39,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for ed2a52e34f710bc9c88f30746da808e9: 2024-12-02T09:20:39,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9. 2024-12-02T09:20:39,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-02T09:20:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37959 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-02T09:20:39,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-02T09:20:39,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 184 msec 2024-12-02T09:20:39,254 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 190 msec 2024-12-02T09:20:39,441 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:39,441 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:40,442 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:40,442 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:41,443 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:41,443 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:42,443 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:42,444 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:43,445 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:43,445 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:44,038 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region ed2a52e34f710bc9c88f30746da808e9, had cached 0 bytes from a total of 14329 2024-12-02T09:20:44,445 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:44,445 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:45,446 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:45,446 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:46,446 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:46,446 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:47,447 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:47,447 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:48,448 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:48,448 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:49,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37959 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T09:20:49,082 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-12-02T09:20:49,084 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C38507%2C1733131197006.1733131249084 2024-12-02T09:20:49,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:49,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:49,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:49,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:49,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:49,091 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/WALs/7c6d666a4939,38507,1733131197006/7c6d666a4939%2C38507%2C1733131197006.1733131239037 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/WALs/7c6d666a4939,38507,1733131197006/7c6d666a4939%2C38507%2C1733131197006.1733131249084 2024-12-02T09:20:49,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741842_1018 (size=2026) 2024-12-02T09:20:49,093 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41289:41289),(127.0.0.1/127.0.0.1:40265:40265)] 2024-12-02T09:20:49,093 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/WALs/7c6d666a4939,38507,1733131197006/7c6d666a4939%2C38507%2C1733131197006.1733131239037 is not closed yet, will try archiving it next time 2024-12-02T09:20:49,094 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-02T09:20:49,094 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T09:20:49,094 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T09:20:49,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:20:49,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:20:49,094 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T09:20:49,094 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-02T09:20:49,094 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2133765893, stopped=false 2024-12-02T09:20:49,094 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7c6d666a4939,37959,1733131196794 2024-12-02T09:20:49,095 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/WALs/7c6d666a4939,38507,1733131197006/7c6d666a4939%2C38507%2C1733131197006.1733131228954 to hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/oldWALs/7c6d666a4939%2C38507%2C1733131197006.1733131228954 2024-12-02T09:20:49,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741842_1018 (size=2026) 2024-12-02T09:20:49,111 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38507-0x1009a4938820001, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T09:20:49,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T09:20:49,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:20:49,111 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38507-0x1009a4938820001, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:20:49,111 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T09:20:49,112 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T09:20:49,112 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T09:20:49,112 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:20:49,112 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:20:49,112 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7c6d666a4939,38507,1733131197006' ***** 2024-12-02T09:20:49,112 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-02T09:20:49,112 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38507-0x1009a4938820001, quorum=127.0.0.1:63670, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:20:49,112 INFO [RS:0;7c6d666a4939:38507 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T09:20:49,112 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T09:20:49,112 INFO [RS:0;7c6d666a4939:38507 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-02T09:20:49,112 INFO [RS:0;7c6d666a4939:38507 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-02T09:20:49,112 INFO [RS:0;7c6d666a4939:38507 {}] regionserver.HRegionServer(3091): Received CLOSE for ed2a52e34f710bc9c88f30746da808e9 2024-12-02T09:20:49,113 INFO [RS:0;7c6d666a4939:38507 {}] regionserver.HRegionServer(959): stopping server 7c6d666a4939,38507,1733131197006 2024-12-02T09:20:49,113 INFO [RS:0;7c6d666a4939:38507 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T09:20:49,113 INFO [RS:0;7c6d666a4939:38507 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7c6d666a4939:38507. 2024-12-02T09:20:49,113 DEBUG [RS:0;7c6d666a4939:38507 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T09:20:49,113 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ed2a52e34f710bc9c88f30746da808e9, disabling compactions & flushes 2024-12-02T09:20:49,113 DEBUG [RS:0;7c6d666a4939:38507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:20:49,113 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9. 2024-12-02T09:20:49,113 INFO [RS:0;7c6d666a4939:38507 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T09:20:49,113 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9. 2024-12-02T09:20:49,113 INFO [RS:0;7c6d666a4939:38507 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T09:20:49,113 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9. after waiting 0 ms 2024-12-02T09:20:49,113 INFO [RS:0;7c6d666a4939:38507 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T09:20:49,113 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9. 2024-12-02T09:20:49,113 INFO [RS:0;7c6d666a4939:38507 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-02T09:20:49,113 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing ed2a52e34f710bc9c88f30746da808e9 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-12-02T09:20:49,113 INFO [RS:0;7c6d666a4939:38507 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-02T09:20:49,113 DEBUG [RS:0;7c6d666a4939:38507 {}] regionserver.HRegionServer(1325): Online Regions={ed2a52e34f710bc9c88f30746da808e9=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9., 1588230740=hbase:meta,,1.1588230740} 2024-12-02T09:20:49,113 DEBUG [RS:0;7c6d666a4939:38507 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, ed2a52e34f710bc9c88f30746da808e9 2024-12-02T09:20:49,113 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T09:20:49,113 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T09:20:49,113 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T09:20:49,113 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T09:20:49,113 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T09:20:49,113 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB 2024-12-02T09:20:49,117 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/.tmp/info/111bb8b937304499b089b9f108821e06 is 1080, key is row0001/info:/1733131249083/Put/seqid=0 2024-12-02T09:20:49,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741845_1021 (size=6033) 2024-12-02T09:20:49,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741845_1021 (size=6033) 2024-12-02T09:20:49,122 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/.tmp/info/111bb8b937304499b089b9f108821e06 2024-12-02T09:20:49,128 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/hbase/meta/1588230740/.tmp/info/13db2d732f8a4fecab837ce029440399 is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9./info:regioninfo/1733131199054/Put/seqid=0 2024-12-02T09:20:49,128 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/.tmp/info/111bb8b937304499b089b9f108821e06 as hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/info/111bb8b937304499b089b9f108821e06 2024-12-02T09:20:49,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741846_1022 (size=7308) 2024-12-02T09:20:49,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741846_1022 (size=7308) 2024-12-02T09:20:49,134 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/hbase/meta/1588230740/.tmp/info/13db2d732f8a4fecab837ce029440399 2024-12-02T09:20:49,135 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/info/111bb8b937304499b089b9f108821e06, entries=1, sequenceid=22, filesize=5.9 K 2024-12-02T09:20:49,136 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for ed2a52e34f710bc9c88f30746da808e9 in 23ms, sequenceid=22, compaction requested=true 2024-12-02T09:20:49,137 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/info/4cbc915f1ac14d0aa83318051c2b5295, hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/info/bc779e7cf3b34b85bd8c83941a9c36e0, hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/info/d751e309654d4fd1ac6d599c02ca2dfd] to archive 2024-12-02T09:20:49,138 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T09:20:49,139 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/info/4cbc915f1ac14d0aa83318051c2b5295 to hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/info/4cbc915f1ac14d0aa83318051c2b5295 2024-12-02T09:20:49,141 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/info/bc779e7cf3b34b85bd8c83941a9c36e0 to hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/info/bc779e7cf3b34b85bd8c83941a9c36e0 2024-12-02T09:20:49,142 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/info/d751e309654d4fd1ac6d599c02ca2dfd to hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/info/d751e309654d4fd1ac6d599c02ca2dfd 2024-12-02T09:20:49,142 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=7c6d666a4939:37959 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-12-02T09:20:49,142 WARN [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [4cbc915f1ac14d0aa83318051c2b5295=6033, bc779e7cf3b34b85bd8c83941a9c36e0=6033, d751e309654d4fd1ac6d599c02ca2dfd=6033] 2024-12-02T09:20:49,146 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/ed2a52e34f710bc9c88f30746da808e9/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1 2024-12-02T09:20:49,147 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9. 2024-12-02T09:20:49,147 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ed2a52e34f710bc9c88f30746da808e9: Waiting for close lock at 1733131249113Running coprocessor pre-close hooks at 1733131249113Disabling compacts and flushes for region at 1733131249113Disabling writes for close at 1733131249113Obtaining lock to block concurrent updates at 1733131249113Preparing flush snapshotting stores in ed2a52e34f710bc9c88f30746da808e9 at 1733131249113Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1733131249113Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9. at 1733131249114 (+1 ms)Flushing ed2a52e34f710bc9c88f30746da808e9/info: creating writer at 1733131249114Flushing ed2a52e34f710bc9c88f30746da808e9/info: appending metadata at 1733131249116 (+2 ms)Flushing ed2a52e34f710bc9c88f30746da808e9/info: closing flushed file at 1733131249116Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70d0ae9e: reopening flushed file at 1733131249127 (+11 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for ed2a52e34f710bc9c88f30746da808e9 in 23ms, sequenceid=22, compaction requested=true at 1733131249136 (+9 ms)Writing region close event to WAL at 1733131249143 (+7 ms)Running coprocessor post-close hooks at 1733131249146 (+3 ms)Closed at 1733131249147 (+1 ms) 2024-12-02T09:20:49,147 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733131198693.ed2a52e34f710bc9c88f30746da808e9. 2024-12-02T09:20:49,152 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/hbase/meta/1588230740/.tmp/ns/bdfef01141444f00b014af92ec8f9e90 is 43, key is default/ns:d/1733131198640/Put/seqid=0 2024-12-02T09:20:49,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741847_1023 (size=5153) 2024-12-02T09:20:49,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741847_1023 (size=5153) 2024-12-02T09:20:49,157 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/hbase/meta/1588230740/.tmp/ns/bdfef01141444f00b014af92ec8f9e90 2024-12-02T09:20:49,174 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/hbase/meta/1588230740/.tmp/table/9387ef473355486bbbc9a99cd84fc3b8 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1733131199065/Put/seqid=0 2024-12-02T09:20:49,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741848_1024 (size=5508) 2024-12-02T09:20:49,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741848_1024 (size=5508) 2024-12-02T09:20:49,179 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/hbase/meta/1588230740/.tmp/table/9387ef473355486bbbc9a99cd84fc3b8 2024-12-02T09:20:49,184 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/hbase/meta/1588230740/.tmp/info/13db2d732f8a4fecab837ce029440399 as hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/hbase/meta/1588230740/info/13db2d732f8a4fecab837ce029440399 2024-12-02T09:20:49,190 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/hbase/meta/1588230740/info/13db2d732f8a4fecab837ce029440399, entries=10, sequenceid=11, filesize=7.1 K 2024-12-02T09:20:49,191 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/hbase/meta/1588230740/.tmp/ns/bdfef01141444f00b014af92ec8f9e90 as hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/hbase/meta/1588230740/ns/bdfef01141444f00b014af92ec8f9e90 2024-12-02T09:20:49,196 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/hbase/meta/1588230740/ns/bdfef01141444f00b014af92ec8f9e90, entries=2, sequenceid=11, filesize=5.0 K 2024-12-02T09:20:49,197 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/hbase/meta/1588230740/.tmp/table/9387ef473355486bbbc9a99cd84fc3b8 as hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/hbase/meta/1588230740/table/9387ef473355486bbbc9a99cd84fc3b8 2024-12-02T09:20:49,202 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/hbase/meta/1588230740/table/9387ef473355486bbbc9a99cd84fc3b8, entries=2, sequenceid=11, filesize=5.4 K 2024-12-02T09:20:49,203 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 90ms, sequenceid=11, compaction requested=false 2024-12-02T09:20:49,207 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-02T09:20:49,208 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T09:20:49,208 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T09:20:49,208 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733131249113Running coprocessor pre-close hooks at 1733131249113Disabling compacts and flushes for region at 1733131249113Disabling writes for close at 1733131249113Obtaining lock to block concurrent updates at 1733131249113Preparing flush snapshotting stores in 1588230740 at 1733131249113Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1733131249114 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733131249114Flushing 1588230740/info: creating writer at 1733131249114Flushing 1588230740/info: appending metadata at 1733131249128 (+14 ms)Flushing 1588230740/info: closing flushed file at 1733131249128Flushing 1588230740/ns: creating writer at 1733131249139 (+11 ms)Flushing 1588230740/ns: appending metadata at 1733131249152 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1733131249152Flushing 1588230740/table: creating writer at 1733131249162 (+10 ms)Flushing 1588230740/table: appending metadata at 1733131249173 (+11 ms)Flushing 1588230740/table: closing flushed file at 1733131249174 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70c0a415: reopening flushed file at 1733131249183 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d1d4fc3: reopening flushed file at 1733131249190 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64383e0e: reopening flushed file at 1733131249196 (+6 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 90ms, sequenceid=11, compaction requested=false at 1733131249203 (+7 ms)Writing region close event to WAL at 1733131249204 (+1 ms)Running coprocessor post-close hooks at 1733131249208 (+4 ms)Closed at 1733131249208 2024-12-02T09:20:49,208 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-02T09:20:49,313 INFO [RS:0;7c6d666a4939:38507 {}] regionserver.HRegionServer(976): stopping server 7c6d666a4939,38507,1733131197006; all regions closed. 2024-12-02T09:20:49,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:49,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:49,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:49,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:49,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:49,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741834_1010 (size=3306) 2024-12-02T09:20:49,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741834_1010 (size=3306) 2024-12-02T09:20:49,448 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:49,448 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:49,731 DEBUG [RS:0;7c6d666a4939:38507 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/oldWALs 2024-12-02T09:20:49,731 INFO [RS:0;7c6d666a4939:38507 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7c6d666a4939%2C38507%2C1733131197006.meta:.meta(num 1733131198561) 2024-12-02T09:20:49,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:49,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:49,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:49,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:49,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:49,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741844_1020 (size=1252) 2024-12-02T09:20:49,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741844_1020 (size=1252) 2024-12-02T09:20:49,736 DEBUG [RS:0;7c6d666a4939:38507 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/oldWALs 2024-12-02T09:20:49,736 INFO [RS:0;7c6d666a4939:38507 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7c6d666a4939%2C38507%2C1733131197006:(num 1733131249084) 2024-12-02T09:20:49,736 DEBUG [RS:0;7c6d666a4939:38507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:20:49,736 INFO [RS:0;7c6d666a4939:38507 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T09:20:49,736 INFO [RS:0;7c6d666a4939:38507 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T09:20:49,736 INFO [RS:0;7c6d666a4939:38507 {}] hbase.ChoreService(370): Chore service for: regionserver/7c6d666a4939:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-02T09:20:49,736 INFO [RS:0;7c6d666a4939:38507 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T09:20:49,737 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T09:20:49,737 INFO [RS:0;7c6d666a4939:38507 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38507 2024-12-02T09:20:49,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T09:20:49,791 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38507-0x1009a4938820001, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7c6d666a4939,38507,1733131197006 2024-12-02T09:20:49,792 INFO [RS:0;7c6d666a4939:38507 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T09:20:49,792 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7c6d666a4939,38507,1733131197006] 2024-12-02T09:20:49,819 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7c6d666a4939,38507,1733131197006 already deleted, retry=false 2024-12-02T09:20:49,819 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7c6d666a4939,38507,1733131197006 expired; onlineServers=0 2024-12-02T09:20:49,819 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7c6d666a4939,37959,1733131196794' ***** 2024-12-02T09:20:49,819 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-02T09:20:49,820 INFO [M:0;7c6d666a4939:37959 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T09:20:49,820 INFO [M:0;7c6d666a4939:37959 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T09:20:49,820 DEBUG [M:0;7c6d666a4939:37959 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-02T09:20:49,820 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-02T09:20:49,820 DEBUG [M:0;7c6d666a4939:37959 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-02T09:20:49,820 DEBUG [master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.small.0-1733131197931 {}] cleaner.HFileCleaner(306): Exit Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.small.0-1733131197931,5,FailOnTimeoutGroup] 2024-12-02T09:20:49,820 DEBUG [master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.large.0-1733131197927 {}] cleaner.HFileCleaner(306): Exit Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.large.0-1733131197927,5,FailOnTimeoutGroup] 2024-12-02T09:20:49,820 INFO [M:0;7c6d666a4939:37959 {}] hbase.ChoreService(370): Chore service for: master/7c6d666a4939:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-02T09:20:49,820 INFO [M:0;7c6d666a4939:37959 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T09:20:49,820 DEBUG [M:0;7c6d666a4939:37959 {}] master.HMaster(1795): Stopping service threads 2024-12-02T09:20:49,820 INFO [M:0;7c6d666a4939:37959 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-02T09:20:49,820 INFO [M:0;7c6d666a4939:37959 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T09:20:49,820 INFO [M:0;7c6d666a4939:37959 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T09:20:49,821 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-02T09:20:49,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-02T09:20:49,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:20:49,828 DEBUG [M:0;7c6d666a4939:37959 {}] zookeeper.ZKUtil(347): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-02T09:20:49,828 WARN [M:0;7c6d666a4939:37959 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-02T09:20:49,829 INFO [M:0;7c6d666a4939:37959 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/.lastflushedseqids 2024-12-02T09:20:49,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741849_1025 (size=130) 2024-12-02T09:20:49,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741849_1025 (size=130) 2024-12-02T09:20:49,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T09:20:49,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-02T09:20:49,834 INFO [M:0;7c6d666a4939:37959 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T09:20:49,834 INFO [M:0;7c6d666a4939:37959 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-02T09:20:49,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-12-02T09:20:49,834 DEBUG [M:0;7c6d666a4939:37959 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T09:20:49,834 INFO [M:0;7c6d666a4939:37959 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:20:49,834 DEBUG [M:0;7c6d666a4939:37959 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:20:49,834 DEBUG [M:0;7c6d666a4939:37959 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T09:20:49,834 DEBUG [M:0;7c6d666a4939:37959 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:20:49,834 INFO [M:0;7c6d666a4939:37959 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.60 KB heapSize=55.01 KB 2024-12-02T09:20:49,847 DEBUG [M:0;7c6d666a4939:37959 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0da9afc2f06d458cbc4a74471a17023c is 82, key is hbase:meta,,1/info:regioninfo/1733131198593/Put/seqid=0 2024-12-02T09:20:49,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741850_1026 (size=5672) 2024-12-02T09:20:49,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741850_1026 (size=5672) 2024-12-02T09:20:49,852 INFO [M:0;7c6d666a4939:37959 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0da9afc2f06d458cbc4a74471a17023c 2024-12-02T09:20:49,869 DEBUG [M:0;7c6d666a4939:37959 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/602b8a760c7d48f6b493bfeb97812287 is 798, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733131199069/Put/seqid=0 2024-12-02T09:20:49,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741851_1027 (size=7824) 2024-12-02T09:20:49,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741851_1027 (size=7824) 2024-12-02T09:20:49,873 INFO [M:0;7c6d666a4939:37959 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.00 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/602b8a760c7d48f6b493bfeb97812287 2024-12-02T09:20:49,877 INFO [M:0;7c6d666a4939:37959 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 602b8a760c7d48f6b493bfeb97812287 2024-12-02T09:20:49,889 DEBUG [M:0;7c6d666a4939:37959 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7785e356a0c442cba83e2e47699da488 is 69, key is 7c6d666a4939,38507,1733131197006/rs:state/1733131198011/Put/seqid=0 2024-12-02T09:20:49,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741852_1028 (size=5156) 2024-12-02T09:20:49,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741852_1028 (size=5156) 2024-12-02T09:20:49,894 INFO [M:0;7c6d666a4939:37959 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7785e356a0c442cba83e2e47699da488 2024-12-02T09:20:49,911 DEBUG [M:0;7c6d666a4939:37959 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d457cd98ff874e6f907a110a61c5718f is 52, key is load_balancer_on/state:d/1733131198689/Put/seqid=0 2024-12-02T09:20:49,911 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38507-0x1009a4938820001, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:20:49,911 INFO [RS:0;7c6d666a4939:38507 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T09:20:49,911 DEBUG [pool-700-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38507-0x1009a4938820001, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:20:49,911 INFO [RS:0;7c6d666a4939:38507 {}] regionserver.HRegionServer(1031): Exiting; stopping=7c6d666a4939,38507,1733131197006; zookeeper connection closed. 2024-12-02T09:20:49,912 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5d329cf7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5d329cf7 2024-12-02T09:20:49,912 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-02T09:20:49,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741853_1029 (size=5056) 2024-12-02T09:20:49,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741853_1029 (size=5056) 2024-12-02T09:20:49,916 INFO [M:0;7c6d666a4939:37959 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d457cd98ff874e6f907a110a61c5718f 2024-12-02T09:20:49,920 DEBUG [M:0;7c6d666a4939:37959 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0da9afc2f06d458cbc4a74471a17023c as hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0da9afc2f06d458cbc4a74471a17023c 2024-12-02T09:20:49,924 INFO [M:0;7c6d666a4939:37959 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0da9afc2f06d458cbc4a74471a17023c, entries=8, sequenceid=121, filesize=5.5 K 2024-12-02T09:20:49,925 DEBUG [M:0;7c6d666a4939:37959 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/602b8a760c7d48f6b493bfeb97812287 as hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/602b8a760c7d48f6b493bfeb97812287 2024-12-02T09:20:49,930 INFO [M:0;7c6d666a4939:37959 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 602b8a760c7d48f6b493bfeb97812287 2024-12-02T09:20:49,930 INFO [M:0;7c6d666a4939:37959 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/602b8a760c7d48f6b493bfeb97812287, entries=14, sequenceid=121, filesize=7.6 K 2024-12-02T09:20:49,931 DEBUG [M:0;7c6d666a4939:37959 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7785e356a0c442cba83e2e47699da488 as hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7785e356a0c442cba83e2e47699da488 2024-12-02T09:20:49,935 INFO [M:0;7c6d666a4939:37959 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7785e356a0c442cba83e2e47699da488, entries=1, sequenceid=121, filesize=5.0 K 2024-12-02T09:20:49,936 DEBUG [M:0;7c6d666a4939:37959 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d457cd98ff874e6f907a110a61c5718f as hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/d457cd98ff874e6f907a110a61c5718f 2024-12-02T09:20:49,942 INFO [M:0;7c6d666a4939:37959 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43733/user/jenkins/test-data/7d1b6bc4-6ece-03c2-1b8b-7de6cc93e9c5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/d457cd98ff874e6f907a110a61c5718f, entries=1, sequenceid=121, filesize=4.9 K 2024-12-02T09:20:49,943 INFO [M:0;7c6d666a4939:37959 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.60 KB/44647, heapSize ~54.95 KB/56264, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 109ms, sequenceid=121, compaction requested=false 2024-12-02T09:20:49,949 INFO [M:0;7c6d666a4939:37959 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:20:49,949 DEBUG [M:0;7c6d666a4939:37959 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733131249834Disabling compacts and flushes for region at 1733131249834Disabling writes for close at 1733131249834Obtaining lock to block concurrent updates at 1733131249834Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733131249834Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44647, getHeapSize=56264, getOffHeapSize=0, getCellsCount=140 at 1733131249835 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733131249835Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733131249835Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733131249846 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733131249846Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733131249856 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733131249868 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733131249868Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733131249877 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733131249888 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733131249889 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733131249898 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733131249910 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733131249910Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56060ef3: reopening flushed file at 1733131249919 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ef7fb04: reopening flushed file at 1733131249924 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76ac6773: reopening flushed file at 1733131249930 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31038ea8: reopening flushed file at 1733131249935 (+5 ms)Finished flush of dataSize ~43.60 KB/44647, heapSize ~54.95 KB/56264, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 109ms, sequenceid=121, compaction requested=false at 1733131249943 (+8 ms)Writing region close event to WAL at 1733131249949 (+6 ms)Closed at 1733131249949 2024-12-02T09:20:49,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:49,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:49,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:49,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:49,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:20:49,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41417 is added to blk_1073741830_1006 (size=53044) 2024-12-02T09:20:49,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36365 is added to blk_1073741830_1006 (size=53044) 2024-12-02T09:20:49,953 INFO [M:0;7c6d666a4939:37959 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-02T09:20:49,953 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T09:20:49,953 INFO [M:0;7c6d666a4939:37959 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37959 2024-12-02T09:20:49,953 INFO [M:0;7c6d666a4939:37959 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T09:20:50,032 INFO [regionserver/7c6d666a4939:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T09:20:50,061 INFO [M:0;7c6d666a4939:37959 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T09:20:50,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:20:50,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37959-0x1009a4938820000, quorum=127.0.0.1:63670, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:20:50,064 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@52c57427{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:20:50,064 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2ade404a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:20:50,064 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:20:50,064 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6580d65d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:20:50,064 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72521dfa{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/hadoop.log.dir/,STOPPED} 2024-12-02T09:20:50,065 WARN [BP-1010008577-172.17.0.3-1733131195238 heartbeating to localhost/127.0.0.1:43733 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T09:20:50,065 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T09:20:50,065 WARN [BP-1010008577-172.17.0.3-1733131195238 heartbeating to localhost/127.0.0.1:43733 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1010008577-172.17.0.3-1733131195238 (Datanode Uuid d6878948-8e77-4140-9d19-62a18a0a7156) service to localhost/127.0.0.1:43733 2024-12-02T09:20:50,065 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T09:20:50,066 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/cluster_06bbeb12-f01b-3b5f-4167-1dd52b3a1a69/data/data3/current/BP-1010008577-172.17.0.3-1733131195238 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:20:50,066 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/cluster_06bbeb12-f01b-3b5f-4167-1dd52b3a1a69/data/data4/current/BP-1010008577-172.17.0.3-1733131195238 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:20:50,066 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T09:20:50,073 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@bff2115{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:20:50,073 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3a2777cc{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:20:50,073 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:20:50,074 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2f8e2aa3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:20:50,074 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@8ac3c66{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/hadoop.log.dir/,STOPPED} 2024-12-02T09:20:50,075 WARN [BP-1010008577-172.17.0.3-1733131195238 heartbeating to localhost/127.0.0.1:43733 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T09:20:50,075 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T09:20:50,075 WARN [BP-1010008577-172.17.0.3-1733131195238 heartbeating to localhost/127.0.0.1:43733 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1010008577-172.17.0.3-1733131195238 (Datanode Uuid 6ee299e8-cd70-4970-be41-f0ef12e8ad5f) service to localhost/127.0.0.1:43733 2024-12-02T09:20:50,075 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T09:20:50,076 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/cluster_06bbeb12-f01b-3b5f-4167-1dd52b3a1a69/data/data1/current/BP-1010008577-172.17.0.3-1733131195238 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:20:50,076 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/cluster_06bbeb12-f01b-3b5f-4167-1dd52b3a1a69/data/data2/current/BP-1010008577-172.17.0.3-1733131195238 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:20:50,076 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T09:20:50,082 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@9b25e94{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T09:20:50,082 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3659c62{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:20:50,082 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:20:50,082 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@144fb880{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:20:50,083 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4ff483aa{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/hadoop.log.dir/,STOPPED} 2024-12-02T09:20:50,087 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-02T09:20:50,104 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-02T09:20:50,112 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=204 (was 179) Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:43733 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-36-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43733 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43733 from jenkins.hfs.5 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-35-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:43733 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43733 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-35-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-36-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-35-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-34-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-36-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-34-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43733 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43733 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: HMaster-EventLoopGroup-12-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-34-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-37-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-37-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43733 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-37-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=483 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=222 (was 229), ProcessCount=11 (was 11), AvailableMemoryMB=1169 (was 677) - AvailableMemoryMB LEAK? - 2024-12-02T09:20:50,120 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=204, OpenFileDescriptor=483, MaxFileDescriptor=1048576, SystemLoadAverage=222, ProcessCount=11, AvailableMemoryMB=1170 2024-12-02T09:20:50,120 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-02T09:20:50,120 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/hadoop.log.dir so I do NOT create it in target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce 2024-12-02T09:20:50,120 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b1933971-f6de-b51b-8146-8dff4d087525/hadoop.tmp.dir so I do NOT create it in target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce 2024-12-02T09:20:50,120 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/cluster_ba17a36c-de3c-0319-27d7-59df5dbdde8b, deleteOnExit=true 2024-12-02T09:20:50,120 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-02T09:20:50,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/test.cache.data in system properties and HBase conf 2024-12-02T09:20:50,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/hadoop.tmp.dir in system properties and HBase conf 2024-12-02T09:20:50,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/hadoop.log.dir in system properties and HBase conf 2024-12-02T09:20:50,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-02T09:20:50,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-02T09:20:50,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-02T09:20:50,121 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-02T09:20:50,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-02T09:20:50,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-02T09:20:50,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-02T09:20:50,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T09:20:50,122 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-02T09:20:50,122 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-02T09:20:50,122 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T09:20:50,122 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T09:20:50,122 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-02T09:20:50,122 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/nfs.dump.dir in system properties and HBase conf 2024-12-02T09:20:50,122 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/java.io.tmpdir in system properties and HBase conf 2024-12-02T09:20:50,122 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T09:20:50,122 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-02T09:20:50,122 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-02T09:20:50,132 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-02T09:20:50,353 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:20:50,359 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:20:50,368 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:20:50,368 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:20:50,368 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-02T09:20:50,370 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:20:50,371 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@811037{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:20:50,371 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1d428ad5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:20:50,449 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:50,449 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:50,499 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@35a5806e{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/java.io.tmpdir/jetty-localhost-43653-hadoop-hdfs-3_4_1-tests_jar-_-any-12923579825440514072/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T09:20:50,501 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@191b8d86{HTTP/1.1, (http/1.1)}{localhost:43653} 2024-12-02T09:20:50,501 INFO [Time-limited test {}] server.Server(415): Started @246982ms 2024-12-02T09:20:50,519 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-02T09:20:50,830 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:20:50,836 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:20:50,843 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:20:50,843 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:20:50,844 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T09:20:50,846 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@208716aa{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:20:50,846 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70aed17c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:20:50,939 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5eab25ce{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/java.io.tmpdir/jetty-localhost-41881-hadoop-hdfs-3_4_1-tests_jar-_-any-8459491611046536696/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:20:50,939 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5773e0ea{HTTP/1.1, (http/1.1)}{localhost:41881} 2024-12-02T09:20:50,939 INFO [Time-limited test {}] server.Server(415): Started @247420ms 2024-12-02T09:20:50,940 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T09:20:50,967 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:20:50,970 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:20:50,971 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:20:50,971 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:20:50,971 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T09:20:50,971 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@31f94774{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:20:50,972 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@369abda8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:20:51,102 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1d3b5490{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/java.io.tmpdir/jetty-localhost-45485-hadoop-hdfs-3_4_1-tests_jar-_-any-12573113271075731479/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:20:51,102 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@41b3b520{HTTP/1.1, (http/1.1)}{localhost:45485} 2024-12-02T09:20:51,102 INFO [Time-limited test {}] server.Server(415): Started @247583ms 2024-12-02T09:20:51,103 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T09:20:51,449 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:51,450 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:52,039 WARN [Thread-1964 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/cluster_ba17a36c-de3c-0319-27d7-59df5dbdde8b/data/data2/current/BP-1798352675-172.17.0.3-1733131250135/current, will proceed with Du for space computation calculation, 2024-12-02T09:20:52,039 WARN [Thread-1963 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/cluster_ba17a36c-de3c-0319-27d7-59df5dbdde8b/data/data1/current/BP-1798352675-172.17.0.3-1733131250135/current, will proceed with Du for space computation calculation, 2024-12-02T09:20:52,057 WARN [Thread-1927 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T09:20:52,059 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd46483b5a0935662 with lease ID 0xfc8d38e26ea2d79: Processing first storage report for DS-91f71770-f6fd-42c4-87e3-2f1940097b3d from datanode DatanodeRegistration(127.0.0.1:42255, datanodeUuid=b905a314-56b7-453b-afa4-4a480bcd3fea, infoPort=46257, infoSecurePort=0, ipcPort=43089, storageInfo=lv=-57;cid=testClusterID;nsid=1049762271;c=1733131250135) 2024-12-02T09:20:52,060 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd46483b5a0935662 with lease ID 0xfc8d38e26ea2d79: from storage DS-91f71770-f6fd-42c4-87e3-2f1940097b3d node DatanodeRegistration(127.0.0.1:42255, datanodeUuid=b905a314-56b7-453b-afa4-4a480bcd3fea, infoPort=46257, infoSecurePort=0, ipcPort=43089, storageInfo=lv=-57;cid=testClusterID;nsid=1049762271;c=1733131250135), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:20:52,060 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd46483b5a0935662 with lease ID 0xfc8d38e26ea2d79: Processing first storage report for DS-07344a98-4042-4ea5-868a-0dd8b796f604 from datanode DatanodeRegistration(127.0.0.1:42255, datanodeUuid=b905a314-56b7-453b-afa4-4a480bcd3fea, infoPort=46257, infoSecurePort=0, ipcPort=43089, storageInfo=lv=-57;cid=testClusterID;nsid=1049762271;c=1733131250135) 2024-12-02T09:20:52,060 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd46483b5a0935662 with lease ID 0xfc8d38e26ea2d79: from storage DS-07344a98-4042-4ea5-868a-0dd8b796f604 node DatanodeRegistration(127.0.0.1:42255, datanodeUuid=b905a314-56b7-453b-afa4-4a480bcd3fea, infoPort=46257, infoSecurePort=0, ipcPort=43089, storageInfo=lv=-57;cid=testClusterID;nsid=1049762271;c=1733131250135), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:20:52,068 WARN [Thread-1974 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/cluster_ba17a36c-de3c-0319-27d7-59df5dbdde8b/data/data3/current/BP-1798352675-172.17.0.3-1733131250135/current, will proceed with Du for space computation calculation, 2024-12-02T09:20:52,068 WARN [Thread-1975 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/cluster_ba17a36c-de3c-0319-27d7-59df5dbdde8b/data/data4/current/BP-1798352675-172.17.0.3-1733131250135/current, will proceed with Du for space computation calculation, 2024-12-02T09:20:52,081 WARN [Thread-1950 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T09:20:52,084 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x55e86242ff856510 with lease ID 0xfc8d38e26ea2d7a: Processing first storage report for DS-a503d90b-a926-4c7d-9669-cb21b9d0c03d from datanode DatanodeRegistration(127.0.0.1:42295, datanodeUuid=bdffc36d-9e37-406e-9f09-f3b04c97f328, infoPort=45143, infoSecurePort=0, ipcPort=44727, storageInfo=lv=-57;cid=testClusterID;nsid=1049762271;c=1733131250135) 2024-12-02T09:20:52,084 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x55e86242ff856510 with lease ID 0xfc8d38e26ea2d7a: from storage DS-a503d90b-a926-4c7d-9669-cb21b9d0c03d node DatanodeRegistration(127.0.0.1:42295, datanodeUuid=bdffc36d-9e37-406e-9f09-f3b04c97f328, infoPort=45143, infoSecurePort=0, ipcPort=44727, storageInfo=lv=-57;cid=testClusterID;nsid=1049762271;c=1733131250135), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-02T09:20:52,084 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x55e86242ff856510 with lease ID 0xfc8d38e26ea2d7a: Processing first storage report for DS-160b5080-2b16-4703-9e0e-f7233120600c from datanode DatanodeRegistration(127.0.0.1:42295, datanodeUuid=bdffc36d-9e37-406e-9f09-f3b04c97f328, infoPort=45143, infoSecurePort=0, ipcPort=44727, storageInfo=lv=-57;cid=testClusterID;nsid=1049762271;c=1733131250135) 2024-12-02T09:20:52,084 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x55e86242ff856510 with lease ID 0xfc8d38e26ea2d7a: from storage DS-160b5080-2b16-4703-9e0e-f7233120600c node DatanodeRegistration(127.0.0.1:42295, datanodeUuid=bdffc36d-9e37-406e-9f09-f3b04c97f328, infoPort=45143, infoSecurePort=0, ipcPort=44727, storageInfo=lv=-57;cid=testClusterID;nsid=1049762271;c=1733131250135), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:20:52,128 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce 2024-12-02T09:20:52,133 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/cluster_ba17a36c-de3c-0319-27d7-59df5dbdde8b/zookeeper_0, clientPort=62506, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/cluster_ba17a36c-de3c-0319-27d7-59df5dbdde8b/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/cluster_ba17a36c-de3c-0319-27d7-59df5dbdde8b/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-02T09:20:52,135 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62506 2024-12-02T09:20:52,136 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:20:52,137 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:20:52,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741825_1001 (size=7) 2024-12-02T09:20:52,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741825_1001 (size=7) 2024-12-02T09:20:52,450 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:52,450 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:52,556 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38 with version=8 2024-12-02T09:20:52,556 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/hbase-staging 2024-12-02T09:20:52,558 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7c6d666a4939:0 server-side Connection retries=45 2024-12-02T09:20:52,558 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:20:52,558 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T09:20:52,558 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T09:20:52,558 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:20:52,558 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T09:20:52,558 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-02T09:20:52,559 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T09:20:52,559 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39695 2024-12-02T09:20:52,561 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39695 connecting to ZooKeeper ensemble=127.0.0.1:62506 2024-12-02T09:20:52,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:396950x0, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T09:20:52,602 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39695-0x1009a4a10d80000 connected 2024-12-02T09:20:52,669 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:20:52,670 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:20:52,672 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:20:52,672 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38, hbase.cluster.distributed=false 2024-12-02T09:20:52,673 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T09:20:52,674 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39695 2024-12-02T09:20:52,674 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39695 2024-12-02T09:20:52,674 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39695 2024-12-02T09:20:52,675 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39695 2024-12-02T09:20:52,675 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39695 2024-12-02T09:20:52,687 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7c6d666a4939:0 server-side Connection retries=45 2024-12-02T09:20:52,687 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:20:52,687 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T09:20:52,687 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T09:20:52,687 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:20:52,687 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T09:20:52,687 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T09:20:52,687 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T09:20:52,688 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37787 2024-12-02T09:20:52,689 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37787 connecting to ZooKeeper ensemble=127.0.0.1:62506 2024-12-02T09:20:52,689 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:20:52,691 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:20:52,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:377870x0, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T09:20:52,702 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37787-0x1009a4a10d80001 connected 2024-12-02T09:20:52,702 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37787-0x1009a4a10d80001, quorum=127.0.0.1:62506, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:20:52,703 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T09:20:52,703 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T09:20:52,704 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37787-0x1009a4a10d80001, quorum=127.0.0.1:62506, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T09:20:52,704 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37787-0x1009a4a10d80001, quorum=127.0.0.1:62506, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T09:20:52,705 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37787 2024-12-02T09:20:52,705 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37787 2024-12-02T09:20:52,705 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37787 2024-12-02T09:20:52,705 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37787 2024-12-02T09:20:52,705 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37787 2024-12-02T09:20:52,715 DEBUG [M:0;7c6d666a4939:39695 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7c6d666a4939:39695 2024-12-02T09:20:52,716 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7c6d666a4939,39695,1733131252558 2024-12-02T09:20:52,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:20:52,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1009a4a10d80001, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:20:52,725 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7c6d666a4939,39695,1733131252558 2024-12-02T09:20:52,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1009a4a10d80001, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T09:20:52,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:20:52,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1009a4a10d80001, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:20:52,736 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-02T09:20:52,736 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7c6d666a4939,39695,1733131252558 from backup master directory 2024-12-02T09:20:52,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7c6d666a4939,39695,1733131252558 2024-12-02T09:20:52,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1009a4a10d80001, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:20:52,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:20:52,744 WARN [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T09:20:52,744 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7c6d666a4939,39695,1733131252558 2024-12-02T09:20:52,747 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/hbase.id] with ID: 84b40b69-315c-4af5-b9c4-6465b52594f6 2024-12-02T09:20:52,747 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/.tmp/hbase.id 2024-12-02T09:20:52,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741826_1002 (size=42) 2024-12-02T09:20:52,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741826_1002 (size=42) 2024-12-02T09:20:52,753 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/.tmp/hbase.id]:[hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/hbase.id] 2024-12-02T09:20:52,763 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:20:52,763 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-02T09:20:52,764 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-12-02T09:20:52,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:20:52,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1009a4a10d80001, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:20:52,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741827_1003 (size=196) 2024-12-02T09:20:52,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741827_1003 (size=196) 2024-12-02T09:20:52,780 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T09:20:52,781 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-02T09:20:52,781 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-02T09:20:52,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741828_1004 (size=1189) 2024-12-02T09:20:52,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741828_1004 (size=1189) 2024-12-02T09:20:52,788 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/data/master/store 2024-12-02T09:20:52,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741829_1005 (size=34) 2024-12-02T09:20:52,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741829_1005 (size=34) 2024-12-02T09:20:52,795 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:20:52,795 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T09:20:52,795 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:20:52,795 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:20:52,795 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T09:20:52,795 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:20:52,795 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:20:52,795 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733131252795Disabling compacts and flushes for region at 1733131252795Disabling writes for close at 1733131252795Writing region close event to WAL at 1733131252795Closed at 1733131252795 2024-12-02T09:20:52,796 WARN [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/data/master/store/.initializing 2024-12-02T09:20:52,796 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/WALs/7c6d666a4939,39695,1733131252558 2024-12-02T09:20:52,799 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c6d666a4939%2C39695%2C1733131252558, suffix=, logDir=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/WALs/7c6d666a4939,39695,1733131252558, archiveDir=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/oldWALs, maxLogs=10 2024-12-02T09:20:52,799 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C39695%2C1733131252558.1733131252799 2024-12-02T09:20:52,804 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/WALs/7c6d666a4939,39695,1733131252558/7c6d666a4939%2C39695%2C1733131252558.1733131252799 2024-12-02T09:20:52,804 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46257:46257),(127.0.0.1/127.0.0.1:45143:45143)] 2024-12-02T09:20:52,805 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-02T09:20:52,805 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:20:52,805 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:20:52,805 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:20:52,807 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:20:52,808 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-02T09:20:52,808 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:20:52,808 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:20:52,808 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:20:52,809 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-02T09:20:52,809 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:20:52,810 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:20:52,810 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:20:52,811 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-02T09:20:52,811 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:20:52,811 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:20:52,811 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:20:52,812 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-02T09:20:52,812 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:20:52,813 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:20:52,813 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:20:52,813 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:20:52,814 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:20:52,815 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:20:52,815 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:20:52,815 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T09:20:52,816 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:20:52,818 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T09:20:52,819 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=812665, jitterRate=0.033357322216033936}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T09:20:52,819 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733131252805Initializing all the Stores at 1733131252806 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131252806Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131252806Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131252806Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131252806Cleaning up temporary data from old regions at 1733131252815 (+9 ms)Region opened successfully at 1733131252819 (+4 ms) 2024-12-02T09:20:52,819 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-02T09:20:52,822 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@811b517, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7c6d666a4939/172.17.0.3:0 2024-12-02T09:20:52,823 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-02T09:20:52,823 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-02T09:20:52,823 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-02T09:20:52,823 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T09:20:52,824 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T09:20:52,824 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-02T09:20:52,824 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-02T09:20:52,826 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-02T09:20:52,827 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-02T09:20:52,866 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-02T09:20:52,867 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-02T09:20:52,868 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-02T09:20:52,908 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-02T09:20:52,909 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-02T09:20:52,911 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-02T09:20:52,919 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-02T09:20:52,920 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-02T09:20:52,927 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-02T09:20:52,929 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-02T09:20:52,935 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-02T09:20:52,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1009a4a10d80001, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T09:20:52,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T09:20:52,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1009a4a10d80001, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:20:52,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:20:52,944 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7c6d666a4939,39695,1733131252558, sessionid=0x1009a4a10d80000, setting cluster-up flag (Was=false) 2024-12-02T09:20:52,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:20:52,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1009a4a10d80001, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:20:52,985 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-02T09:20:52,986 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7c6d666a4939,39695,1733131252558 2024-12-02T09:20:53,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1009a4a10d80001, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:20:53,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:20:53,027 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-02T09:20:53,031 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7c6d666a4939,39695,1733131252558 2024-12-02T09:20:53,032 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-02T09:20:53,034 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T09:20:53,034 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-02T09:20:53,034 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-02T09:20:53,034 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7c6d666a4939,39695,1733131252558 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-02T09:20:53,035 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:20:53,035 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:20:53,035 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:20:53,035 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:20:53,035 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7c6d666a4939:0, corePoolSize=10, maxPoolSize=10 2024-12-02T09:20:53,035 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:20:53,036 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7c6d666a4939:0, corePoolSize=2, maxPoolSize=2 2024-12-02T09:20:53,036 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:20:53,038 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733131283038 2024-12-02T09:20:53,038 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-02T09:20:53,038 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-02T09:20:53,038 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-02T09:20:53,038 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-02T09:20:53,038 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T09:20:53,038 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-02T09:20:53,038 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-02T09:20:53,038 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-02T09:20:53,038 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T09:20:53,039 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-02T09:20:53,039 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-02T09:20:53,039 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-02T09:20:53,039 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:20:53,039 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-02T09:20:53,040 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-02T09:20:53,040 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-02T09:20:53,041 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.large.0-1733131253040,5,FailOnTimeoutGroup] 2024-12-02T09:20:53,046 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.small.0-1733131253041,5,FailOnTimeoutGroup] 2024-12-02T09:20:53,046 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T09:20:53,046 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-02T09:20:53,046 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-02T09:20:53,046 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-02T09:20:53,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741831_1007 (size=1321) 2024-12-02T09:20:53,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741831_1007 (size=1321) 2024-12-02T09:20:53,052 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-02T09:20:53,052 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38 2024-12-02T09:20:53,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741832_1008 (size=32) 2024-12-02T09:20:53,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741832_1008 (size=32) 2024-12-02T09:20:53,058 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:20:53,059 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T09:20:53,060 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T09:20:53,061 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:20:53,061 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:20:53,061 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T09:20:53,062 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T09:20:53,062 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:20:53,062 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:20:53,062 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T09:20:53,063 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T09:20:53,063 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:20:53,064 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:20:53,064 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T09:20:53,065 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T09:20:53,065 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:20:53,065 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:20:53,065 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T09:20:53,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/hbase/meta/1588230740 2024-12-02T09:20:53,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/hbase/meta/1588230740 2024-12-02T09:20:53,067 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T09:20:53,067 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T09:20:53,067 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T09:20:53,068 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T09:20:53,070 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T09:20:53,070 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=825601, jitterRate=0.04980659484863281}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T09:20:53,071 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733131253059Initializing all the Stores at 1733131253059Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131253059Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131253059Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131253059Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131253059Cleaning up temporary data from old regions at 1733131253067 (+8 ms)Region opened successfully at 1733131253071 (+4 ms) 2024-12-02T09:20:53,071 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T09:20:53,071 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T09:20:53,071 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T09:20:53,071 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T09:20:53,071 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T09:20:53,071 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T09:20:53,071 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733131253071Disabling compacts and flushes for region at 1733131253071Disabling writes for close at 1733131253071Writing region close event to WAL at 1733131253071Closed at 1733131253071 2024-12-02T09:20:53,073 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T09:20:53,073 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-02T09:20:53,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-02T09:20:53,074 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T09:20:53,075 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-02T09:20:53,107 INFO [RS:0;7c6d666a4939:37787 {}] regionserver.HRegionServer(746): ClusterId : 84b40b69-315c-4af5-b9c4-6465b52594f6 2024-12-02T09:20:53,107 DEBUG [RS:0;7c6d666a4939:37787 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T09:20:53,117 DEBUG [RS:0;7c6d666a4939:37787 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T09:20:53,117 DEBUG [RS:0;7c6d666a4939:37787 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T09:20:53,128 DEBUG [RS:0;7c6d666a4939:37787 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T09:20:53,128 DEBUG [RS:0;7c6d666a4939:37787 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44455fc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7c6d666a4939/172.17.0.3:0 2024-12-02T09:20:53,137 DEBUG [RS:0;7c6d666a4939:37787 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7c6d666a4939:37787 2024-12-02T09:20:53,137 INFO [RS:0;7c6d666a4939:37787 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T09:20:53,137 INFO [RS:0;7c6d666a4939:37787 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T09:20:53,137 DEBUG [RS:0;7c6d666a4939:37787 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T09:20:53,137 INFO [RS:0;7c6d666a4939:37787 {}] regionserver.HRegionServer(2659): reportForDuty to master=7c6d666a4939,39695,1733131252558 with port=37787, startcode=1733131252687 2024-12-02T09:20:53,138 DEBUG [RS:0;7c6d666a4939:37787 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T09:20:53,139 INFO [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47283, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T09:20:53,139 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39695 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7c6d666a4939,37787,1733131252687 2024-12-02T09:20:53,140 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39695 {}] master.ServerManager(517): Registering regionserver=7c6d666a4939,37787,1733131252687 2024-12-02T09:20:53,141 DEBUG [RS:0;7c6d666a4939:37787 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38 2024-12-02T09:20:53,141 DEBUG [RS:0;7c6d666a4939:37787 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:32923 2024-12-02T09:20:53,141 DEBUG [RS:0;7c6d666a4939:37787 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T09:20:53,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T09:20:53,152 DEBUG [RS:0;7c6d666a4939:37787 {}] zookeeper.ZKUtil(111): regionserver:37787-0x1009a4a10d80001, quorum=127.0.0.1:62506, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7c6d666a4939,37787,1733131252687 2024-12-02T09:20:53,152 WARN [RS:0;7c6d666a4939:37787 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T09:20:53,152 INFO [RS:0;7c6d666a4939:37787 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-02T09:20:53,153 DEBUG [RS:0;7c6d666a4939:37787 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/WALs/7c6d666a4939,37787,1733131252687 2024-12-02T09:20:53,153 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7c6d666a4939,37787,1733131252687] 2024-12-02T09:20:53,155 INFO [RS:0;7c6d666a4939:37787 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T09:20:53,157 INFO [RS:0;7c6d666a4939:37787 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T09:20:53,157 INFO [RS:0;7c6d666a4939:37787 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T09:20:53,157 INFO [RS:0;7c6d666a4939:37787 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:20:53,157 INFO [RS:0;7c6d666a4939:37787 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T09:20:53,158 INFO [RS:0;7c6d666a4939:37787 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T09:20:53,158 INFO [RS:0;7c6d666a4939:37787 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T09:20:53,158 DEBUG [RS:0;7c6d666a4939:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:20:53,158 DEBUG [RS:0;7c6d666a4939:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:20:53,158 DEBUG [RS:0;7c6d666a4939:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:20:53,158 DEBUG [RS:0;7c6d666a4939:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:20:53,158 DEBUG [RS:0;7c6d666a4939:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:20:53,158 DEBUG [RS:0;7c6d666a4939:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7c6d666a4939:0, corePoolSize=2, maxPoolSize=2 2024-12-02T09:20:53,158 DEBUG [RS:0;7c6d666a4939:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:20:53,158 DEBUG [RS:0;7c6d666a4939:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:20:53,158 DEBUG [RS:0;7c6d666a4939:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:20:53,158 DEBUG [RS:0;7c6d666a4939:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:20:53,158 DEBUG [RS:0;7c6d666a4939:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:20:53,158 DEBUG [RS:0;7c6d666a4939:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:20:53,158 DEBUG [RS:0;7c6d666a4939:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7c6d666a4939:0, corePoolSize=3, maxPoolSize=3 2024-12-02T09:20:53,158 DEBUG [RS:0;7c6d666a4939:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0, corePoolSize=3, maxPoolSize=3 2024-12-02T09:20:53,159 INFO [RS:0;7c6d666a4939:37787 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T09:20:53,159 INFO [RS:0;7c6d666a4939:37787 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T09:20:53,159 INFO [RS:0;7c6d666a4939:37787 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:20:53,159 INFO [RS:0;7c6d666a4939:37787 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T09:20:53,159 INFO [RS:0;7c6d666a4939:37787 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T09:20:53,159 INFO [RS:0;7c6d666a4939:37787 {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,37787,1733131252687-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T09:20:53,174 INFO [RS:0;7c6d666a4939:37787 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T09:20:53,174 INFO [RS:0;7c6d666a4939:37787 {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,37787,1733131252687-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:20:53,174 INFO [RS:0;7c6d666a4939:37787 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:20:53,174 INFO [RS:0;7c6d666a4939:37787 {}] regionserver.Replication(171): 7c6d666a4939,37787,1733131252687 started 2024-12-02T09:20:53,190 INFO [RS:0;7c6d666a4939:37787 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:20:53,190 INFO [RS:0;7c6d666a4939:37787 {}] regionserver.HRegionServer(1482): Serving as 7c6d666a4939,37787,1733131252687, RpcServer on 7c6d666a4939/172.17.0.3:37787, sessionid=0x1009a4a10d80001 2024-12-02T09:20:53,190 DEBUG [RS:0;7c6d666a4939:37787 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T09:20:53,190 DEBUG [RS:0;7c6d666a4939:37787 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7c6d666a4939,37787,1733131252687 2024-12-02T09:20:53,190 DEBUG [RS:0;7c6d666a4939:37787 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7c6d666a4939,37787,1733131252687' 2024-12-02T09:20:53,190 DEBUG [RS:0;7c6d666a4939:37787 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T09:20:53,191 DEBUG [RS:0;7c6d666a4939:37787 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T09:20:53,191 DEBUG [RS:0;7c6d666a4939:37787 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T09:20:53,191 DEBUG [RS:0;7c6d666a4939:37787 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T09:20:53,191 DEBUG [RS:0;7c6d666a4939:37787 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7c6d666a4939,37787,1733131252687 2024-12-02T09:20:53,191 DEBUG [RS:0;7c6d666a4939:37787 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7c6d666a4939,37787,1733131252687' 2024-12-02T09:20:53,191 DEBUG [RS:0;7c6d666a4939:37787 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T09:20:53,191 DEBUG [RS:0;7c6d666a4939:37787 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T09:20:53,192 DEBUG [RS:0;7c6d666a4939:37787 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T09:20:53,192 INFO [RS:0;7c6d666a4939:37787 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T09:20:53,192 INFO [RS:0;7c6d666a4939:37787 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T09:20:53,225 WARN [7c6d666a4939:39695 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-02T09:20:53,294 INFO [RS:0;7c6d666a4939:37787 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c6d666a4939%2C37787%2C1733131252687, suffix=, logDir=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/WALs/7c6d666a4939,37787,1733131252687, archiveDir=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/oldWALs, maxLogs=32 2024-12-02T09:20:53,294 INFO [RS:0;7c6d666a4939:37787 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C37787%2C1733131252687.1733131253294 2024-12-02T09:20:53,299 INFO [RS:0;7c6d666a4939:37787 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/WALs/7c6d666a4939,37787,1733131252687/7c6d666a4939%2C37787%2C1733131252687.1733131253294 2024-12-02T09:20:53,300 DEBUG [RS:0;7c6d666a4939:37787 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45143:45143),(127.0.0.1/127.0.0.1:46257:46257)] 2024-12-02T09:20:53,450 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:53,450 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:53,475 DEBUG [7c6d666a4939:39695 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-02T09:20:53,476 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7c6d666a4939,37787,1733131252687 2024-12-02T09:20:53,477 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7c6d666a4939,37787,1733131252687, state=OPENING 2024-12-02T09:20:53,508 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-02T09:20:53,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1009a4a10d80001, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:20:53,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:20:53,519 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T09:20:53,519 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:20:53,519 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:20:53,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7c6d666a4939,37787,1733131252687}] 2024-12-02T09:20:53,672 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T09:20:53,673 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57947, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T09:20:53,677 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-02T09:20:53,677 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-02T09:20:53,678 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c6d666a4939%2C37787%2C1733131252687.meta, suffix=.meta, logDir=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/WALs/7c6d666a4939,37787,1733131252687, archiveDir=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/oldWALs, maxLogs=32 2024-12-02T09:20:53,679 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C37787%2C1733131252687.meta.1733131253679.meta 2024-12-02T09:20:53,683 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/WALs/7c6d666a4939,37787,1733131252687/7c6d666a4939%2C37787%2C1733131252687.meta.1733131253679.meta 2024-12-02T09:20:53,684 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46257:46257),(127.0.0.1/127.0.0.1:45143:45143)] 2024-12-02T09:20:53,685 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-02T09:20:53,685 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-02T09:20:53,685 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-02T09:20:53,685 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-02T09:20:53,685 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-02T09:20:53,685 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:20:53,686 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-02T09:20:53,686 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-02T09:20:53,687 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T09:20:53,687 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T09:20:53,687 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:20:53,688 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:20:53,688 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T09:20:53,688 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T09:20:53,688 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:20:53,689 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:20:53,689 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T09:20:53,689 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T09:20:53,689 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:20:53,690 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:20:53,690 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T09:20:53,690 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T09:20:53,690 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:20:53,691 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:20:53,691 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T09:20:53,691 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/hbase/meta/1588230740 2024-12-02T09:20:53,692 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/hbase/meta/1588230740 2024-12-02T09:20:53,693 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T09:20:53,693 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T09:20:53,694 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T09:20:53,695 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T09:20:53,696 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=713129, jitterRate=-0.0932101458311081}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T09:20:53,696 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-02T09:20:53,696 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733131253686Writing region info on filesystem at 1733131253686Initializing all the Stores at 1733131253686Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131253686Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131253686Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131253686Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131253686Cleaning up temporary data from old regions at 1733131253693 (+7 ms)Running coprocessor post-open hooks at 1733131253696 (+3 ms)Region opened successfully at 1733131253696 2024-12-02T09:20:53,697 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733131253671 2024-12-02T09:20:53,699 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-02T09:20:53,699 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-02T09:20:53,700 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7c6d666a4939,37787,1733131252687 2024-12-02T09:20:53,701 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7c6d666a4939,37787,1733131252687, state=OPEN 2024-12-02T09:20:53,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T09:20:53,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1009a4a10d80001, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T09:20:53,729 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7c6d666a4939,37787,1733131252687 2024-12-02T09:20:53,729 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:20:53,729 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:20:53,732 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-02T09:20:53,732 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7c6d666a4939,37787,1733131252687 in 210 msec 2024-12-02T09:20:53,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-02T09:20:53,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 659 msec 2024-12-02T09:20:53,735 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T09:20:53,735 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-02T09:20:53,736 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T09:20:53,736 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c6d666a4939,37787,1733131252687, seqNum=-1] 2024-12-02T09:20:53,736 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T09:20:53,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44945, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T09:20:53,742 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 709 msec 2024-12-02T09:20:53,742 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733131253742, completionTime=-1 2024-12-02T09:20:53,743 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-02T09:20:53,743 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T09:20:53,744 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-02T09:20:53,744 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733131313744 2024-12-02T09:20:53,744 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733131373744 2024-12-02T09:20:53,744 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec 2024-12-02T09:20:53,745 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,39695,1733131252558-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:20:53,745 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,39695,1733131252558-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:20:53,745 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,39695,1733131252558-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:20:53,745 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7c6d666a4939:39695, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:20:53,745 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-02T09:20:53,745 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-02T09:20:53,747 DEBUG [master/7c6d666a4939:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-02T09:20:53,748 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.004sec 2024-12-02T09:20:53,748 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-02T09:20:53,748 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-02T09:20:53,748 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-02T09:20:53,749 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-02T09:20:53,749 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-02T09:20:53,749 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,39695,1733131252558-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T09:20:53,749 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,39695,1733131252558-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-02T09:20:53,751 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-02T09:20:53,751 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-02T09:20:53,751 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,39695,1733131252558-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:20:53,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fead160, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T09:20:53,808 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c6d666a4939,39695,-1 for getting cluster id 2024-12-02T09:20:53,808 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T09:20:53,809 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '84b40b69-315c-4af5-b9c4-6465b52594f6' 2024-12-02T09:20:53,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T09:20:53,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "84b40b69-315c-4af5-b9c4-6465b52594f6" 2024-12-02T09:20:53,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@533a12fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T09:20:53,810 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c6d666a4939,39695,-1] 2024-12-02T09:20:53,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T09:20:53,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:20:53,812 INFO [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T09:20:53,813 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@335c5487, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T09:20:53,813 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T09:20:53,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c6d666a4939,37787,1733131252687, seqNum=-1] 2024-12-02T09:20:53,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T09:20:53,815 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T09:20:53,817 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7c6d666a4939,39695,1733131252558 2024-12-02T09:20:53,817 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:20:53,820 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-12-02T09:20:53,820 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T09:20:53,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 7c6d666a4939,39695,1733131252558 2024-12-02T09:20:53,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1dab8729 2024-12-02T09:20:53,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T09:20:53,823 INFO [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T09:20:53,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39695 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-12-02T09:20:53,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39695 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-12-02T09:20:53,824 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39695 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T09:20:53,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39695 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling 2024-12-02T09:20:53,826 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T09:20:53,826 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:20:53,826 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39695 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRolling" procId is: 4 2024-12-02T09:20:53,827 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T09:20:53,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T09:20:53,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741835_1011 (size=381) 2024-12-02T09:20:53,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741835_1011 (size=381) 2024-12-02T09:20:53,836 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => adb545fc7773af344dd36d23ae47da66, NAME => 'TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38 2024-12-02T09:20:53,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741836_1012 (size=64) 2024-12-02T09:20:53,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741836_1012 (size=64) 2024-12-02T09:20:53,843 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:20:53,843 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing adb545fc7773af344dd36d23ae47da66, disabling compactions & flushes 2024-12-02T09:20:53,843 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66. 2024-12-02T09:20:53,843 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66. 2024-12-02T09:20:53,843 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66. after waiting 0 ms 2024-12-02T09:20:53,843 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66. 2024-12-02T09:20:53,843 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66. 2024-12-02T09:20:53,843 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for adb545fc7773af344dd36d23ae47da66: Waiting for close lock at 1733131253843Disabling compacts and flushes for region at 1733131253843Disabling writes for close at 1733131253843Writing region close event to WAL at 1733131253843Closed at 1733131253843 2024-12-02T09:20:53,844 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T09:20:53,845 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733131253845"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733131253845"}]},"ts":"1733131253845"} 2024-12-02T09:20:53,847 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T09:20:53,848 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T09:20:53,848 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733131253848"}]},"ts":"1733131253848"} 2024-12-02T09:20:53,851 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta 2024-12-02T09:20:53,851 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=adb545fc7773af344dd36d23ae47da66, ASSIGN}] 2024-12-02T09:20:53,852 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=adb545fc7773af344dd36d23ae47da66, ASSIGN 2024-12-02T09:20:53,853 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=adb545fc7773af344dd36d23ae47da66, ASSIGN; state=OFFLINE, location=7c6d666a4939,37787,1733131252687; forceNewPlan=false, retain=false 2024-12-02T09:20:54,004 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=adb545fc7773af344dd36d23ae47da66, regionState=OPENING, regionLocation=7c6d666a4939,37787,1733131252687 2024-12-02T09:20:54,006 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=adb545fc7773af344dd36d23ae47da66, ASSIGN because future has completed 2024-12-02T09:20:54,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure adb545fc7773af344dd36d23ae47da66, server=7c6d666a4939,37787,1733131252687}] 2024-12-02T09:20:54,147 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,148 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,148 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,148 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,148 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,148 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,149 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,149 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,163 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66. 2024-12-02T09:20:54,163 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => adb545fc7773af344dd36d23ae47da66, NAME => 'TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66.', STARTKEY => '', ENDKEY => ''} 2024-12-02T09:20:54,164 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling adb545fc7773af344dd36d23ae47da66 2024-12-02T09:20:54,164 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:20:54,164 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for adb545fc7773af344dd36d23ae47da66 2024-12-02T09:20:54,164 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for adb545fc7773af344dd36d23ae47da66 2024-12-02T09:20:54,165 INFO [StoreOpener-adb545fc7773af344dd36d23ae47da66-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region adb545fc7773af344dd36d23ae47da66 2024-12-02T09:20:54,166 INFO [StoreOpener-adb545fc7773af344dd36d23ae47da66-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region adb545fc7773af344dd36d23ae47da66 columnFamilyName info 2024-12-02T09:20:54,166 DEBUG [StoreOpener-adb545fc7773af344dd36d23ae47da66-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:20:54,167 INFO [StoreOpener-adb545fc7773af344dd36d23ae47da66-1 {}] regionserver.HStore(327): Store=adb545fc7773af344dd36d23ae47da66/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:20:54,167 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for adb545fc7773af344dd36d23ae47da66 2024-12-02T09:20:54,167 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66 2024-12-02T09:20:54,168 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66 2024-12-02T09:20:54,168 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for adb545fc7773af344dd36d23ae47da66 2024-12-02T09:20:54,168 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for adb545fc7773af344dd36d23ae47da66 2024-12-02T09:20:54,170 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,170 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for adb545fc7773af344dd36d23ae47da66 2024-12-02T09:20:54,170 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,170 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,170 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,170 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,170 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,172 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T09:20:54,172 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened adb545fc7773af344dd36d23ae47da66; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=781938, jitterRate=-0.00571480393409729}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T09:20:54,172 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for adb545fc7773af344dd36d23ae47da66 2024-12-02T09:20:54,173 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for adb545fc7773af344dd36d23ae47da66: Running coprocessor pre-open hook at 1733131254164Writing region info on filesystem at 1733131254164Initializing all the Stores at 1733131254165 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131254165Cleaning up temporary data from old regions at 1733131254168 (+3 ms)Running coprocessor post-open hooks at 1733131254172 (+4 ms)Region opened successfully at 1733131254172 2024-12-02T09:20:54,173 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,174 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66., pid=6, masterSystemTime=1733131254159 2024-12-02T09:20:54,174 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,174 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,176 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66. 2024-12-02T09:20:54,176 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66. 2024-12-02T09:20:54,176 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,177 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=adb545fc7773af344dd36d23ae47da66, regionState=OPEN, openSeqNum=2, regionLocation=7c6d666a4939,37787,1733131252687 2024-12-02T09:20:54,179 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure adb545fc7773af344dd36d23ae47da66, server=7c6d666a4939,37787,1733131252687 because future has completed 2024-12-02T09:20:54,182 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-02T09:20:54,182 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure adb545fc7773af344dd36d23ae47da66, server=7c6d666a4939,37787,1733131252687 in 173 msec 2024-12-02T09:20:54,184 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-02T09:20:54,185 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=adb545fc7773af344dd36d23ae47da66, ASSIGN in 331 msec 2024-12-02T09:20:54,185 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T09:20:54,185 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733131254185"}]},"ts":"1733131254185"} 2024-12-02T09:20:54,188 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta 2024-12-02T09:20:54,189 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T09:20:54,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 365 msec 2024-12-02T09:20:54,451 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:54,451 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:54,681 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-02T09:20:54,682 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,682 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,682 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,682 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,682 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,682 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,683 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,683 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,698 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,699 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,699 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,699 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,699 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,699 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,702 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,702 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,702 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:54,705 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:55,452 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:55,452 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:56,452 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:56,452 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:57,453 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:57,453 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:58,453 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:58,453 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:59,156 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-02T09:20:59,156 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling' 2024-12-02T09:20:59,454 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:59,454 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:20:59,666 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-02T09:20:59,667 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:59,667 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:59,667 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:59,667 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:59,668 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:59,668 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:59,668 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:59,668 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:59,686 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:59,686 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:59,686 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:59,686 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:59,686 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:59,687 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:59,689 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:59,689 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:59,690 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:59,692 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:20:59,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-02T09:20:59,834 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-02T09:20:59,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T09:20:59,834 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-02T09:20:59,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling 2024-12-02T09:20:59,834 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer 2024-12-02T09:21:00,455 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:00,455 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:01,455 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:01,455 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:02,456 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:02,456 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:03,456 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:03,456 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:03,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T09:21:03,852 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed 2024-12-02T09:21:03,852 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100 2024-12-02T09:21:03,854 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling 2024-12-02T09:21:03,854 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66. 2024-12-02T09:21:03,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66., hostname=7c6d666a4939,37787,1733131252687, seqNum=2] 2024-12-02T09:21:03,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] regionserver.HRegion(8855): Flush requested on adb545fc7773af344dd36d23ae47da66 2024-12-02T09:21:03,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing adb545fc7773af344dd36d23ae47da66 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-02T09:21:03,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/.tmp/info/93921ccb552c4ba0bde26ea5952619f8 is 1080, key is row0001/info:/1733131263857/Put/seqid=0 2024-12-02T09:21:03,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741837_1013 (size=12509) 2024-12-02T09:21:03,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741837_1013 (size=12509) 2024-12-02T09:21:03,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/.tmp/info/93921ccb552c4ba0bde26ea5952619f8 2024-12-02T09:21:03,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/.tmp/info/93921ccb552c4ba0bde26ea5952619f8 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/93921ccb552c4ba0bde26ea5952619f8 2024-12-02T09:21:03,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/93921ccb552c4ba0bde26ea5952619f8, entries=7, sequenceid=11, filesize=12.2 K 2024-12-02T09:21:03,914 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=18.91 KB/19368 for adb545fc7773af344dd36d23ae47da66 in 43ms, sequenceid=11, compaction requested=false 2024-12-02T09:21:03,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for adb545fc7773af344dd36d23ae47da66: 2024-12-02T09:21:03,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] regionserver.HRegion(8855): Flush requested on adb545fc7773af344dd36d23ae47da66 2024-12-02T09:21:03,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing adb545fc7773af344dd36d23ae47da66 1/1 column families, dataSize=21.02 KB heapSize=22.75 KB 2024-12-02T09:21:03,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/.tmp/info/0960d7eb5c7b495cb52e782523729392 is 1080, key is row0008/info:/1733131263872/Put/seqid=0 2024-12-02T09:21:03,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741838_1014 (size=26530) 2024-12-02T09:21:03,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741838_1014 (size=26530) 2024-12-02T09:21:03,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=21.02 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/.tmp/info/0960d7eb5c7b495cb52e782523729392 2024-12-02T09:21:03,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/.tmp/info/0960d7eb5c7b495cb52e782523729392 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/0960d7eb5c7b495cb52e782523729392 2024-12-02T09:21:03,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/0960d7eb5c7b495cb52e782523729392, entries=20, sequenceid=34, filesize=25.9 K 2024-12-02T09:21:03,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~21.02 KB/21520, heapSize ~22.73 KB/23280, currentSize=5.25 KB/5380 for adb545fc7773af344dd36d23ae47da66 in 24ms, sequenceid=34, compaction requested=false 2024-12-02T09:21:03,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for adb545fc7773af344dd36d23ae47da66: 2024-12-02T09:21:03,940 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=38.1 K, sizeToCheck=16.0 K 2024-12-02T09:21:03,940 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-02T09:21:03,940 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/0960d7eb5c7b495cb52e782523729392 because midkey is the same as first or last row 2024-12-02T09:21:04,457 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:04,457 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:05,457 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:05,457 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] regionserver.HRegion(8855): Flush requested on adb545fc7773af344dd36d23ae47da66 2024-12-02T09:21:05,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing adb545fc7773af344dd36d23ae47da66 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-02T09:21:05,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/.tmp/info/6ddc2c99ceca4360bcafb60a39442d6b is 1080, key is row0028/info:/1733131263918/Put/seqid=0 2024-12-02T09:21:05,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741839_1015 (size=12509) 2024-12-02T09:21:05,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741839_1015 (size=12509) 2024-12-02T09:21:05,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=44 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/.tmp/info/6ddc2c99ceca4360bcafb60a39442d6b 2024-12-02T09:21:05,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/.tmp/info/6ddc2c99ceca4360bcafb60a39442d6b as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/6ddc2c99ceca4360bcafb60a39442d6b 2024-12-02T09:21:05,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/6ddc2c99ceca4360bcafb60a39442d6b, entries=7, sequenceid=44, filesize=12.2 K 2024-12-02T09:21:05,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=12.61 KB/12912 for adb545fc7773af344dd36d23ae47da66 in 32ms, sequenceid=44, compaction requested=true 2024-12-02T09:21:05,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for adb545fc7773af344dd36d23ae47da66: 2024-12-02T09:21:05,966 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=50.3 K, sizeToCheck=16.0 K 2024-12-02T09:21:05,966 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-02T09:21:05,966 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/0960d7eb5c7b495cb52e782523729392 because midkey is the same as first or last row 2024-12-02T09:21:05,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store adb545fc7773af344dd36d23ae47da66:info, priority=-2147483648, current under compaction store size is 1 2024-12-02T09:21:05,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:21:05,967 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T09:21:05,968 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 51548 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T09:21:05,968 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1541): adb545fc7773af344dd36d23ae47da66/info is initiating minor compaction (all files) 2024-12-02T09:21:05,968 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of adb545fc7773af344dd36d23ae47da66/info in TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66. 2024-12-02T09:21:05,969 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/93921ccb552c4ba0bde26ea5952619f8, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/0960d7eb5c7b495cb52e782523729392, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/6ddc2c99ceca4360bcafb60a39442d6b] into tmpdir=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/.tmp, totalSize=50.3 K 2024-12-02T09:21:05,969 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93921ccb552c4ba0bde26ea5952619f8, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733131263857 2024-12-02T09:21:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] regionserver.HRegion(8855): Flush requested on adb545fc7773af344dd36d23ae47da66 2024-12-02T09:21:05,970 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0960d7eb5c7b495cb52e782523729392, keycount=20, bloomtype=ROW, size=25.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733131263872 2024-12-02T09:21:05,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing adb545fc7773af344dd36d23ae47da66 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB 2024-12-02T09:21:05,971 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ddc2c99ceca4360bcafb60a39442d6b, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1733131263918 2024-12-02T09:21:05,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/.tmp/info/540cc5538d184d778fdfc2bdb4fe9cb7 is 1080, key is row0035/info:/1733131265935/Put/seqid=0 2024-12-02T09:21:05,988 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): adb545fc7773af344dd36d23ae47da66#info#compaction#59 average throughput is 11.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T09:21:05,988 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/.tmp/info/d38db702c69145a6857621f36141f03e is 1080, key is row0001/info:/1733131263857/Put/seqid=0 2024-12-02T09:21:06,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741840_1016 (size=18987) 2024-12-02T09:21:06,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741840_1016 (size=18987) 2024-12-02T09:21:06,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/.tmp/info/540cc5538d184d778fdfc2bdb4fe9cb7 2024-12-02T09:21:06,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741841_1017 (size=41747) 2024-12-02T09:21:06,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741841_1017 (size=41747) 2024-12-02T09:21:06,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/.tmp/info/540cc5538d184d778fdfc2bdb4fe9cb7 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/540cc5538d184d778fdfc2bdb4fe9cb7 2024-12-02T09:21:06,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/540cc5538d184d778fdfc2bdb4fe9cb7, entries=13, sequenceid=60, filesize=18.5 K 2024-12-02T09:21:06,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=adb545fc7773af344dd36d23ae47da66, server=7c6d666a4939,37787,1733131252687 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T09:21:06,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=16.81 KB/17216 for adb545fc7773af344dd36d23ae47da66 in 55ms, sequenceid=60, compaction requested=false 2024-12-02T09:21:06,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for adb545fc7773af344dd36d23ae47da66: 2024-12-02T09:21:06,025 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=68.9 K, sizeToCheck=16.0 K 2024-12-02T09:21:06,025 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-02T09:21:06,025 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/0960d7eb5c7b495cb52e782523729392 because midkey is the same as first or last row 2024-12-02T09:21:06,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.3:48016 deadline: 1733131276023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=adb545fc7773af344dd36d23ae47da66, server=7c6d666a4939,37787,1733131252687 2024-12-02T09:21:06,031 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/.tmp/info/d38db702c69145a6857621f36141f03e as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/d38db702c69145a6857621f36141f03e 2024-12-02T09:21:06,037 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in adb545fc7773af344dd36d23ae47da66/info of adb545fc7773af344dd36d23ae47da66 into d38db702c69145a6857621f36141f03e(size=40.8 K), total size for store is 59.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T09:21:06,037 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for adb545fc7773af344dd36d23ae47da66: 2024-12-02T09:21:06,037 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66., storeName=adb545fc7773af344dd36d23ae47da66/info, priority=13, startTime=1733131265967; duration=0sec 2024-12-02T09:21:06,038 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=59.3 K, sizeToCheck=16.0 K 2024-12-02T09:21:06,038 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-02T09:21:06,038 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/d38db702c69145a6857621f36141f03e because midkey is the same as first or last row 2024-12-02T09:21:06,038 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=59.3 K, sizeToCheck=16.0 K 2024-12-02T09:21:06,038 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-02T09:21:06,038 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/d38db702c69145a6857621f36141f03e because midkey is the same as first or last row 2024-12-02T09:21:06,038 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=59.3 K, sizeToCheck=16.0 K 2024-12-02T09:21:06,038 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-02T09:21:06,038 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/d38db702c69145a6857621f36141f03e because midkey is the same as first or last row 2024-12-02T09:21:06,038 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:21:06,038 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: adb545fc7773af344dd36d23ae47da66:info 2024-12-02T09:21:06,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66., hostname=7c6d666a4939,37787,1733131252687, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66., hostname=7c6d666a4939,37787,1733131252687, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=adb545fc7773af344dd36d23ae47da66, server=7c6d666a4939,37787,1733131252687 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T09:21:06,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66., hostname=7c6d666a4939,37787,1733131252687, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=adb545fc7773af344dd36d23ae47da66, server=7c6d666a4939,37787,1733131252687 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T09:21:06,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66., hostname=7c6d666a4939,37787,1733131252687, seqNum=2 because the exception is null or not the one we care about 2024-12-02T09:21:06,458 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:06,458 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:07,459 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:07,459 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:08,459 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:08,459 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:09,460 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:09,461 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:10,461 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:10,461 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:11,462 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:11,462 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:12,463 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:12,463 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:13,464 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:13,464 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:14,465 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:14,465 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:15,466 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:15,466 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:16,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] regionserver.HRegion(8855): Flush requested on adb545fc7773af344dd36d23ae47da66 2024-12-02T09:21:16,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing adb545fc7773af344dd36d23ae47da66 1/1 column families, dataSize=17.86 KB heapSize=19.38 KB 2024-12-02T09:21:16,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/.tmp/info/0c465207804b4946b4d2a66cf1ed8e76 is 1080, key is row0048/info:/1733131265972/Put/seqid=0 2024-12-02T09:21:16,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741842_1018 (size=23299) 2024-12-02T09:21:16,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741842_1018 (size=23299) 2024-12-02T09:21:16,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.86 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/.tmp/info/0c465207804b4946b4d2a66cf1ed8e76 2024-12-02T09:21:16,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/.tmp/info/0c465207804b4946b4d2a66cf1ed8e76 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/0c465207804b4946b4d2a66cf1ed8e76 2024-12-02T09:21:16,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/0c465207804b4946b4d2a66cf1ed8e76, entries=17, sequenceid=81, filesize=22.8 K 2024-12-02T09:21:16,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~17.86 KB/18292, heapSize ~19.36 KB/19824, currentSize=0 B/0 for adb545fc7773af344dd36d23ae47da66 in 43ms, sequenceid=81, compaction requested=true 2024-12-02T09:21:16,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for adb545fc7773af344dd36d23ae47da66: 2024-12-02T09:21:16,147 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=82.1 K, sizeToCheck=16.0 K 2024-12-02T09:21:16,147 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-02T09:21:16,147 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/d38db702c69145a6857621f36141f03e because midkey is the same as first or last row 2024-12-02T09:21:16,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store adb545fc7773af344dd36d23ae47da66:info, priority=-2147483648, current under compaction store size is 1 2024-12-02T09:21:16,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:21:16,147 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T09:21:16,149 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 84033 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T09:21:16,149 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1541): adb545fc7773af344dd36d23ae47da66/info is initiating minor compaction (all files) 2024-12-02T09:21:16,149 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of adb545fc7773af344dd36d23ae47da66/info in TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66. 2024-12-02T09:21:16,149 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/d38db702c69145a6857621f36141f03e, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/540cc5538d184d778fdfc2bdb4fe9cb7, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/0c465207804b4946b4d2a66cf1ed8e76] into tmpdir=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/.tmp, totalSize=82.1 K 2024-12-02T09:21:16,150 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting d38db702c69145a6857621f36141f03e, keycount=34, bloomtype=ROW, size=40.8 K, encoding=NONE, compression=NONE, seqNum=44, earliestPutTs=1733131263857 2024-12-02T09:21:16,150 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting 540cc5538d184d778fdfc2bdb4fe9cb7, keycount=13, bloomtype=ROW, size=18.5 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733131265935 2024-12-02T09:21:16,151 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c465207804b4946b4d2a66cf1ed8e76, keycount=17, bloomtype=ROW, size=22.8 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733131265972 2024-12-02T09:21:16,169 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): adb545fc7773af344dd36d23ae47da66#info#compaction#61 average throughput is 21.89 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T09:21:16,169 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/.tmp/info/bb27cad513c94baca1abc85930a76f31 is 1080, key is row0001/info:/1733131263857/Put/seqid=0 2024-12-02T09:21:16,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741843_1019 (size=74301) 2024-12-02T09:21:16,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741843_1019 (size=74301) 2024-12-02T09:21:16,195 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/.tmp/info/bb27cad513c94baca1abc85930a76f31 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/bb27cad513c94baca1abc85930a76f31 2024-12-02T09:21:16,203 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in adb545fc7773af344dd36d23ae47da66/info of adb545fc7773af344dd36d23ae47da66 into bb27cad513c94baca1abc85930a76f31(size=72.6 K), total size for store is 72.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T09:21:16,203 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for adb545fc7773af344dd36d23ae47da66: 2024-12-02T09:21:16,203 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66., storeName=adb545fc7773af344dd36d23ae47da66/info, priority=13, startTime=1733131276147; duration=0sec 2024-12-02T09:21:16,203 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=72.6 K, sizeToCheck=16.0 K 2024-12-02T09:21:16,203 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-02T09:21:16,204 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=72.6 K, sizeToCheck=16.0 K 2024-12-02T09:21:16,204 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-02T09:21:16,204 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=72.6 K, sizeToCheck=16.0 K 2024-12-02T09:21:16,204 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-12-02T09:21:16,205 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:21:16,205 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:21:16,205 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: adb545fc7773af344dd36d23ae47da66:info 2024-12-02T09:21:16,207 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39695 {}] assignment.AssignmentManager(1363): Split request from 7c6d666a4939,37787,1733131252687, parent={ENCODED => adb545fc7773af344dd36d23ae47da66, NAME => 'TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062 2024-12-02T09:21:16,213 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39695 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=7c6d666a4939,37787,1733131252687 2024-12-02T09:21:16,218 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39695 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=adb545fc7773af344dd36d23ae47da66, daughterA=e2c45d5b11370f58eb0c1f4939fe5ffa, daughterB=88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:16,220 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=adb545fc7773af344dd36d23ae47da66, daughterA=e2c45d5b11370f58eb0c1f4939fe5ffa, daughterB=88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:16,220 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=adb545fc7773af344dd36d23ae47da66, daughterA=e2c45d5b11370f58eb0c1f4939fe5ffa, daughterB=88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:16,220 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=adb545fc7773af344dd36d23ae47da66, daughterA=e2c45d5b11370f58eb0c1f4939fe5ffa, daughterB=88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:16,227 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=adb545fc7773af344dd36d23ae47da66, UNASSIGN}] 2024-12-02T09:21:16,229 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=adb545fc7773af344dd36d23ae47da66, UNASSIGN 2024-12-02T09:21:16,231 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=adb545fc7773af344dd36d23ae47da66, regionState=CLOSING, regionLocation=7c6d666a4939,37787,1733131252687 2024-12-02T09:21:16,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=adb545fc7773af344dd36d23ae47da66, UNASSIGN because future has completed 2024-12-02T09:21:16,234 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false 2024-12-02T09:21:16,235 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure adb545fc7773af344dd36d23ae47da66, server=7c6d666a4939,37787,1733131252687}] 2024-12-02T09:21:16,393 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close adb545fc7773af344dd36d23ae47da66 2024-12-02T09:21:16,393 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true 2024-12-02T09:21:16,393 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing adb545fc7773af344dd36d23ae47da66, disabling compactions & flushes 2024-12-02T09:21:16,393 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66. 2024-12-02T09:21:16,393 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66. 2024-12-02T09:21:16,393 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66. after waiting 0 ms 2024-12-02T09:21:16,393 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66. 2024-12-02T09:21:16,394 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/93921ccb552c4ba0bde26ea5952619f8, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/0960d7eb5c7b495cb52e782523729392, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/d38db702c69145a6857621f36141f03e, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/6ddc2c99ceca4360bcafb60a39442d6b, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/540cc5538d184d778fdfc2bdb4fe9cb7, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/0c465207804b4946b4d2a66cf1ed8e76] to archive 2024-12-02T09:21:16,395 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T09:21:16,397 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/93921ccb552c4ba0bde26ea5952619f8 to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/93921ccb552c4ba0bde26ea5952619f8 2024-12-02T09:21:16,398 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/0960d7eb5c7b495cb52e782523729392 to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/0960d7eb5c7b495cb52e782523729392 2024-12-02T09:21:16,400 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/d38db702c69145a6857621f36141f03e to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/d38db702c69145a6857621f36141f03e 2024-12-02T09:21:16,401 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/6ddc2c99ceca4360bcafb60a39442d6b to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/6ddc2c99ceca4360bcafb60a39442d6b 2024-12-02T09:21:16,402 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/540cc5538d184d778fdfc2bdb4fe9cb7 to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/540cc5538d184d778fdfc2bdb4fe9cb7 2024-12-02T09:21:16,403 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/0c465207804b4946b4d2a66cf1ed8e76 to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/0c465207804b4946b4d2a66cf1ed8e76 2024-12-02T09:21:16,408 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/recovered.edits/85.seqid, newMaxSeqId=85, maxSeqId=1 2024-12-02T09:21:16,409 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66. 2024-12-02T09:21:16,409 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for adb545fc7773af344dd36d23ae47da66: Waiting for close lock at 1733131276393Running coprocessor pre-close hooks at 1733131276393Disabling compacts and flushes for region at 1733131276393Disabling writes for close at 1733131276393Writing region close event to WAL at 1733131276405 (+12 ms)Running coprocessor post-close hooks at 1733131276409 (+4 ms)Closed at 1733131276409 2024-12-02T09:21:16,411 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed adb545fc7773af344dd36d23ae47da66 2024-12-02T09:21:16,411 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=adb545fc7773af344dd36d23ae47da66, regionState=CLOSED 2024-12-02T09:21:16,413 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure adb545fc7773af344dd36d23ae47da66, server=7c6d666a4939,37787,1733131252687 because future has completed 2024-12-02T09:21:16,418 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-02T09:21:16,418 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure adb545fc7773af344dd36d23ae47da66, server=7c6d666a4939,37787,1733131252687 in 180 msec 2024-12-02T09:21:16,421 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-02T09:21:16,421 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=adb545fc7773af344dd36d23ae47da66, UNASSIGN in 191 msec 2024-12-02T09:21:16,428 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:21:16,431 INFO [PEWorker-4 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 1 storefiles, region=adb545fc7773af344dd36d23ae47da66, threads=1 2024-12-02T09:21:16,432 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/bb27cad513c94baca1abc85930a76f31 for region: adb545fc7773af344dd36d23ae47da66 2024-12-02T09:21:16,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741844_1020 (size=27) 2024-12-02T09:21:16,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741844_1020 (size=27) 2024-12-02T09:21:16,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741845_1021 (size=27) 2024-12-02T09:21:16,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741845_1021 (size=27) 2024-12-02T09:21:16,456 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/bb27cad513c94baca1abc85930a76f31 for region: adb545fc7773af344dd36d23ae47da66 2024-12-02T09:21:16,459 DEBUG [PEWorker-4 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region adb545fc7773af344dd36d23ae47da66 Daughter A: [hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/e2c45d5b11370f58eb0c1f4939fe5ffa/info/bb27cad513c94baca1abc85930a76f31.adb545fc7773af344dd36d23ae47da66] storefiles, Daughter B: [hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/bb27cad513c94baca1abc85930a76f31.adb545fc7773af344dd36d23ae47da66] storefiles. 2024-12-02T09:21:16,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741846_1022 (size=71) 2024-12-02T09:21:16,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741846_1022 (size=71) 2024-12-02T09:21:16,467 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:16,467 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:16,469 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:21:16,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741847_1023 (size=71) 2024-12-02T09:21:16,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741847_1023 (size=71) 2024-12-02T09:21:16,481 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:21:16,491 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/e2c45d5b11370f58eb0c1f4939fe5ffa/recovered.edits/85.seqid, newMaxSeqId=85, maxSeqId=-1 2024-12-02T09:21:16,494 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/recovered.edits/85.seqid, newMaxSeqId=85, maxSeqId=-1 2024-12-02T09:21:16,497 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733131276497"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1733131276497"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1733131276497"}]},"ts":"1733131276497"} 2024-12-02T09:21:16,497 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733131276213.e2c45d5b11370f58eb0c1f4939fe5ffa.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733131276497"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733131276497"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733131276497"}]},"ts":"1733131276497"} 2024-12-02T09:21:16,497 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733131276497"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733131276497"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733131276497"}]},"ts":"1733131276497"} 2024-12-02T09:21:16,513 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e2c45d5b11370f58eb0c1f4939fe5ffa, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=88bc4495114400d5a5b4dc77ae590483, ASSIGN}] 2024-12-02T09:21:16,515 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e2c45d5b11370f58eb0c1f4939fe5ffa, ASSIGN 2024-12-02T09:21:16,515 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=88bc4495114400d5a5b4dc77ae590483, ASSIGN 2024-12-02T09:21:16,516 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e2c45d5b11370f58eb0c1f4939fe5ffa, ASSIGN; state=SPLITTING_NEW, location=7c6d666a4939,37787,1733131252687; forceNewPlan=false, retain=false 2024-12-02T09:21:16,516 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=88bc4495114400d5a5b4dc77ae590483, ASSIGN; state=SPLITTING_NEW, location=7c6d666a4939,37787,1733131252687; forceNewPlan=false, retain=false 2024-12-02T09:21:16,666 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=88bc4495114400d5a5b4dc77ae590483, regionState=OPENING, regionLocation=7c6d666a4939,37787,1733131252687 2024-12-02T09:21:16,667 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e2c45d5b11370f58eb0c1f4939fe5ffa, regionState=OPENING, regionLocation=7c6d666a4939,37787,1733131252687 2024-12-02T09:21:16,669 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=88bc4495114400d5a5b4dc77ae590483, ASSIGN because future has completed 2024-12-02T09:21:16,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88bc4495114400d5a5b4dc77ae590483, server=7c6d666a4939,37787,1733131252687}] 2024-12-02T09:21:16,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e2c45d5b11370f58eb0c1f4939fe5ffa, ASSIGN because future has completed 2024-12-02T09:21:16,672 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2c45d5b11370f58eb0c1f4939fe5ffa, server=7c6d666a4939,37787,1733131252687}] 2024-12-02T09:21:16,826 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1733131276213.e2c45d5b11370f58eb0c1f4939fe5ffa. 2024-12-02T09:21:16,826 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => e2c45d5b11370f58eb0c1f4939fe5ffa, NAME => 'TestLogRolling-testLogRolling,,1733131276213.e2c45d5b11370f58eb0c1f4939fe5ffa.', STARTKEY => '', ENDKEY => 'row0062'} 2024-12-02T09:21:16,826 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling e2c45d5b11370f58eb0c1f4939fe5ffa 2024-12-02T09:21:16,827 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733131276213.e2c45d5b11370f58eb0c1f4939fe5ffa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:21:16,827 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for e2c45d5b11370f58eb0c1f4939fe5ffa 2024-12-02T09:21:16,827 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for e2c45d5b11370f58eb0c1f4939fe5ffa 2024-12-02T09:21:16,828 INFO [StoreOpener-e2c45d5b11370f58eb0c1f4939fe5ffa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region e2c45d5b11370f58eb0c1f4939fe5ffa 2024-12-02T09:21:16,829 INFO [StoreOpener-e2c45d5b11370f58eb0c1f4939fe5ffa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2c45d5b11370f58eb0c1f4939fe5ffa columnFamilyName info 2024-12-02T09:21:16,829 DEBUG [StoreOpener-e2c45d5b11370f58eb0c1f4939fe5ffa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:21:16,841 DEBUG [StoreOpener-e2c45d5b11370f58eb0c1f4939fe5ffa-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/e2c45d5b11370f58eb0c1f4939fe5ffa/info/bb27cad513c94baca1abc85930a76f31.adb545fc7773af344dd36d23ae47da66->hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/bb27cad513c94baca1abc85930a76f31-bottom 2024-12-02T09:21:16,842 INFO [StoreOpener-e2c45d5b11370f58eb0c1f4939fe5ffa-1 {}] regionserver.HStore(327): Store=e2c45d5b11370f58eb0c1f4939fe5ffa/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:21:16,842 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for e2c45d5b11370f58eb0c1f4939fe5ffa 2024-12-02T09:21:16,842 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/e2c45d5b11370f58eb0c1f4939fe5ffa 2024-12-02T09:21:16,843 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/e2c45d5b11370f58eb0c1f4939fe5ffa 2024-12-02T09:21:16,844 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for e2c45d5b11370f58eb0c1f4939fe5ffa 2024-12-02T09:21:16,844 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for e2c45d5b11370f58eb0c1f4939fe5ffa 2024-12-02T09:21:16,846 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for e2c45d5b11370f58eb0c1f4939fe5ffa 2024-12-02T09:21:16,846 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened e2c45d5b11370f58eb0c1f4939fe5ffa; next sequenceid=86; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=862249, jitterRate=0.0964067280292511}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T09:21:16,847 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e2c45d5b11370f58eb0c1f4939fe5ffa 2024-12-02T09:21:16,847 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for e2c45d5b11370f58eb0c1f4939fe5ffa: Running coprocessor pre-open hook at 1733131276827Writing region info on filesystem at 1733131276827Initializing all the Stores at 1733131276827Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131276827Cleaning up temporary data from old regions at 1733131276844 (+17 ms)Running coprocessor post-open hooks at 1733131276847 (+3 ms)Region opened successfully at 1733131276847 2024-12-02T09:21:16,848 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1733131276213.e2c45d5b11370f58eb0c1f4939fe5ffa., pid=13, masterSystemTime=1733131276823 2024-12-02T09:21:16,848 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store e2c45d5b11370f58eb0c1f4939fe5ffa:info, priority=-2147483648, current under compaction store size is 1 2024-12-02T09:21:16,849 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:21:16,849 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-02T09:21:16,849 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1733131276213.e2c45d5b11370f58eb0c1f4939fe5ffa. 2024-12-02T09:21:16,849 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1541): e2c45d5b11370f58eb0c1f4939fe5ffa/info is initiating minor compaction (all files) 2024-12-02T09:21:16,849 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e2c45d5b11370f58eb0c1f4939fe5ffa/info in TestLogRolling-testLogRolling,,1733131276213.e2c45d5b11370f58eb0c1f4939fe5ffa. 2024-12-02T09:21:16,850 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/e2c45d5b11370f58eb0c1f4939fe5ffa/info/bb27cad513c94baca1abc85930a76f31.adb545fc7773af344dd36d23ae47da66->hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/bb27cad513c94baca1abc85930a76f31-bottom] into tmpdir=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/e2c45d5b11370f58eb0c1f4939fe5ffa/.tmp, totalSize=72.6 K 2024-12-02T09:21:16,850 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb27cad513c94baca1abc85930a76f31.adb545fc7773af344dd36d23ae47da66, keycount=32, bloomtype=ROW, size=72.6 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733131263857 2024-12-02T09:21:16,851 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1733131276213.e2c45d5b11370f58eb0c1f4939fe5ffa. 2024-12-02T09:21:16,851 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1733131276213.e2c45d5b11370f58eb0c1f4939fe5ffa. 2024-12-02T09:21:16,851 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483. 2024-12-02T09:21:16,851 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 88bc4495114400d5a5b4dc77ae590483, NAME => 'TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.', STARTKEY => 'row0062', ENDKEY => ''} 2024-12-02T09:21:16,852 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:16,852 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:21:16,852 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e2c45d5b11370f58eb0c1f4939fe5ffa, regionState=OPEN, openSeqNum=86, regionLocation=7c6d666a4939,37787,1733131252687 2024-12-02T09:21:16,852 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:16,852 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:16,854 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37787 {}] regionserver.HRegion(8855): Flush requested on 1588230740 2024-12-02T09:21:16,854 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-12-02T09:21:16,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.11 KB heapSize=8.96 KB 2024-12-02T09:21:16,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2c45d5b11370f58eb0c1f4939fe5ffa, server=7c6d666a4939,37787,1733131252687 because future has completed 2024-12-02T09:21:16,859 INFO [StoreOpener-88bc4495114400d5a5b4dc77ae590483-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:16,860 INFO [StoreOpener-88bc4495114400d5a5b4dc77ae590483-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88bc4495114400d5a5b4dc77ae590483 columnFamilyName info 2024-12-02T09:21:16,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=10 2024-12-02T09:21:16,860 DEBUG [StoreOpener-88bc4495114400d5a5b4dc77ae590483-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:21:16,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure e2c45d5b11370f58eb0c1f4939fe5ffa, server=7c6d666a4939,37787,1733131252687 in 186 msec 2024-12-02T09:21:16,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e2c45d5b11370f58eb0c1f4939fe5ffa, ASSIGN in 348 msec 2024-12-02T09:21:16,876 DEBUG [StoreOpener-88bc4495114400d5a5b4dc77ae590483-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/bb27cad513c94baca1abc85930a76f31.adb545fc7773af344dd36d23ae47da66->hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/bb27cad513c94baca1abc85930a76f31-top 2024-12-02T09:21:16,876 INFO [StoreOpener-88bc4495114400d5a5b4dc77ae590483-1 {}] regionserver.HStore(327): Store=88bc4495114400d5a5b4dc77ae590483/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:21:16,877 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:16,877 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:16,878 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e2c45d5b11370f58eb0c1f4939fe5ffa#info#compaction#62 average throughput is 20.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T09:21:16,879 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/e2c45d5b11370f58eb0c1f4939fe5ffa/.tmp/info/1f212b0b04c243a8a634a8dc43704ab6 is 1080, key is row0001/info:/1733131263857/Put/seqid=0 2024-12-02T09:21:16,879 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:16,879 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:16,879 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:16,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/hbase/meta/1588230740/.tmp/info/7c109807793047d795f80c022aecb17d is 193, key is TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483./info:regioninfo/1733131276666/Put/seqid=0 2024-12-02T09:21:16,882 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:16,883 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 88bc4495114400d5a5b4dc77ae590483; next sequenceid=86; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=844959, jitterRate=0.07442131638526917}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-02T09:21:16,883 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:16,883 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 88bc4495114400d5a5b4dc77ae590483: Running coprocessor pre-open hook at 1733131276852Writing region info on filesystem at 1733131276852Initializing all the Stores at 1733131276856 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131276856Cleaning up temporary data from old regions at 1733131276879 (+23 ms)Running coprocessor post-open hooks at 1733131276883 (+4 ms)Region opened successfully at 1733131276883 2024-12-02T09:21:16,884 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483., pid=12, masterSystemTime=1733131276823 2024-12-02T09:21:16,884 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store 88bc4495114400d5a5b4dc77ae590483:info, priority=-2147483648, current under compaction store size is 2 2024-12-02T09:21:16,884 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:21:16,884 DEBUG [RS:0;7c6d666a4939:37787-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-02T09:21:16,885 INFO [RS:0;7c6d666a4939:37787-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483. 2024-12-02T09:21:16,885 DEBUG [RS:0;7c6d666a4939:37787-longCompactions-0 {}] regionserver.HStore(1541): 88bc4495114400d5a5b4dc77ae590483/info is initiating minor compaction (all files) 2024-12-02T09:21:16,885 INFO [RS:0;7c6d666a4939:37787-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88bc4495114400d5a5b4dc77ae590483/info in TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483. 2024-12-02T09:21:16,885 INFO [RS:0;7c6d666a4939:37787-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/bb27cad513c94baca1abc85930a76f31.adb545fc7773af344dd36d23ae47da66->hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/bb27cad513c94baca1abc85930a76f31-top] into tmpdir=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp, totalSize=72.6 K 2024-12-02T09:21:16,886 DEBUG [RS:0;7c6d666a4939:37787-longCompactions-0 {}] compactions.Compactor(225): Compacting bb27cad513c94baca1abc85930a76f31.adb545fc7773af344dd36d23ae47da66, keycount=32, bloomtype=ROW, size=72.6 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733131263857 2024-12-02T09:21:16,887 DEBUG [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483. 2024-12-02T09:21:16,887 INFO [RS_OPEN_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483. 2024-12-02T09:21:16,888 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=88bc4495114400d5a5b4dc77ae590483, regionState=OPEN, openSeqNum=86, regionLocation=7c6d666a4939,37787,1733131252687 2024-12-02T09:21:16,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88bc4495114400d5a5b4dc77ae590483, server=7c6d666a4939,37787,1733131252687 because future has completed 2024-12-02T09:21:16,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741848_1024 (size=70862) 2024-12-02T09:21:16,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741848_1024 (size=70862) 2024-12-02T09:21:16,911 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/e2c45d5b11370f58eb0c1f4939fe5ffa/.tmp/info/1f212b0b04c243a8a634a8dc43704ab6 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/e2c45d5b11370f58eb0c1f4939fe5ffa/info/1f212b0b04c243a8a634a8dc43704ab6 2024-12-02T09:21:16,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-02T09:21:16,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 88bc4495114400d5a5b4dc77ae590483, server=7c6d666a4939,37787,1733131252687 in 241 msec 2024-12-02T09:21:16,918 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=7 2024-12-02T09:21:16,918 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=88bc4495114400d5a5b4dc77ae590483, ASSIGN in 401 msec 2024-12-02T09:21:16,919 INFO [RS:0;7c6d666a4939:37787-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88bc4495114400d5a5b4dc77ae590483#info#compaction#64 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T09:21:16,919 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in e2c45d5b11370f58eb0c1f4939fe5ffa/info of e2c45d5b11370f58eb0c1f4939fe5ffa into 1f212b0b04c243a8a634a8dc43704ab6(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T09:21:16,919 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e2c45d5b11370f58eb0c1f4939fe5ffa: 2024-12-02T09:21:16,919 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733131276213.e2c45d5b11370f58eb0c1f4939fe5ffa., storeName=e2c45d5b11370f58eb0c1f4939fe5ffa/info, priority=15, startTime=1733131276848; duration=0sec 2024-12-02T09:21:16,920 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:21:16,920 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e2c45d5b11370f58eb0c1f4939fe5ffa:info 2024-12-02T09:21:16,920 DEBUG [RS:0;7c6d666a4939:37787-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/c3cc2ae7234040bca23de967853e6efd is 1080, key is row0062/info:/1733131266017/Put/seqid=0 2024-12-02T09:21:16,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=adb545fc7773af344dd36d23ae47da66, daughterA=e2c45d5b11370f58eb0c1f4939fe5ffa, daughterB=88bc4495114400d5a5b4dc77ae590483 in 705 msec 2024-12-02T09:21:16,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741849_1025 (size=9847) 2024-12-02T09:21:16,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741849_1025 (size=9847) 2024-12-02T09:21:16,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.92 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/hbase/meta/1588230740/.tmp/info/7c109807793047d795f80c022aecb17d 2024-12-02T09:21:16,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741850_1026 (size=8260) 2024-12-02T09:21:16,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741850_1026 (size=8260) 2024-12-02T09:21:16,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/hbase/meta/1588230740/.tmp/ns/3c8c204618af48f1833bdef8b7b729e3 is 43, key is default/ns:d/1733131253738/Put/seqid=0 2024-12-02T09:21:16,950 DEBUG [RS:0;7c6d666a4939:37787-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/c3cc2ae7234040bca23de967853e6efd as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/c3cc2ae7234040bca23de967853e6efd 2024-12-02T09:21:16,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741851_1027 (size=5153) 2024-12-02T09:21:16,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741851_1027 (size=5153) 2024-12-02T09:21:16,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/hbase/meta/1588230740/.tmp/ns/3c8c204618af48f1833bdef8b7b729e3 2024-12-02T09:21:16,959 INFO [RS:0;7c6d666a4939:37787-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in 88bc4495114400d5a5b4dc77ae590483/info of 88bc4495114400d5a5b4dc77ae590483 into c3cc2ae7234040bca23de967853e6efd(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T09:21:16,960 DEBUG [RS:0;7c6d666a4939:37787-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88bc4495114400d5a5b4dc77ae590483: 2024-12-02T09:21:16,960 INFO [RS:0;7c6d666a4939:37787-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483., storeName=88bc4495114400d5a5b4dc77ae590483/info, priority=15, startTime=1733131276884; duration=0sec 2024-12-02T09:21:16,960 DEBUG [RS:0;7c6d666a4939:37787-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:21:16,960 DEBUG [RS:0;7c6d666a4939:37787-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88bc4495114400d5a5b4dc77ae590483:info 2024-12-02T09:21:16,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/hbase/meta/1588230740/.tmp/table/b6d63499e6cf468e8afa354cb0e0c5a0 is 65, key is TestLogRolling-testLogRolling/table:state/1733131254185/Put/seqid=0 2024-12-02T09:21:16,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741852_1028 (size=5340) 2024-12-02T09:21:16,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741852_1028 (size=5340) 2024-12-02T09:21:16,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/hbase/meta/1588230740/.tmp/table/b6d63499e6cf468e8afa354cb0e0c5a0 2024-12-02T09:21:16,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/hbase/meta/1588230740/.tmp/info/7c109807793047d795f80c022aecb17d as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/hbase/meta/1588230740/info/7c109807793047d795f80c022aecb17d 2024-12-02T09:21:16,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/hbase/meta/1588230740/info/7c109807793047d795f80c022aecb17d, entries=30, sequenceid=17, filesize=9.6 K 2024-12-02T09:21:16,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/hbase/meta/1588230740/.tmp/ns/3c8c204618af48f1833bdef8b7b729e3 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/hbase/meta/1588230740/ns/3c8c204618af48f1833bdef8b7b729e3 2024-12-02T09:21:16,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/hbase/meta/1588230740/ns/3c8c204618af48f1833bdef8b7b729e3, entries=2, sequenceid=17, filesize=5.0 K 2024-12-02T09:21:16,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/hbase/meta/1588230740/.tmp/table/b6d63499e6cf468e8afa354cb0e0c5a0 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/hbase/meta/1588230740/table/b6d63499e6cf468e8afa354cb0e0c5a0 2024-12-02T09:21:17,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/hbase/meta/1588230740/table/b6d63499e6cf468e8afa354cb0e0c5a0, entries=2, sequenceid=17, filesize=5.2 K 2024-12-02T09:21:17,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.11 KB/5234, heapSize ~8.66 KB/8872, currentSize=705 B/705 for 1588230740 in 152ms, sequenceid=17, compaction requested=false 2024-12-02T09:21:17,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-12-02T09:21:17,467 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:17,467 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:18,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.3:48016 deadline: 1733131288104, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66. is not online on 7c6d666a4939,37787,1733131252687 2024-12-02T09:21:18,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66., hostname=7c6d666a4939,37787,1733131252687, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66., hostname=7c6d666a4939,37787,1733131252687, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66. is not online on 7c6d666a4939,37787,1733131252687 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T09:21:18,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66., hostname=7c6d666a4939,37787,1733131252687, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66. is not online on 7c6d666a4939,37787,1733131252687 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T09:21:18,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1733131253823.adb545fc7773af344dd36d23ae47da66., hostname=7c6d666a4939,37787,1733131252687, seqNum=2 from cache 2024-12-02T09:21:18,468 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:18,468 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:19,468 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:19,468 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:20,469 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:20,469 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:21,409 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,410 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,410 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,410 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,410 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,410 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,411 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,411 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,438 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,438 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,438 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,439 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,439 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,439 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,442 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,442 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,442 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,444 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,470 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:21,470 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:21,950 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-12-02T09:21:21,951 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,951 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,951 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,952 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,952 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,952 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,952 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,952 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,973 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,973 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,974 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,974 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,974 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,975 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,978 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,978 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,978 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:21,982 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-12-02T09:21:22,128 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T09:21:22,470 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:22,470 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:23,471 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:23,471 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:24,471 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:24,471 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:25,472 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:25,472 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:26,472 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:26,472 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:27,473 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:27,473 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:28,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0065', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483., hostname=7c6d666a4939,37787,1733131252687, seqNum=86] 2024-12-02T09:21:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] regionserver.HRegion(8855): Flush requested on 88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:28,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88bc4495114400d5a5b4dc77ae590483 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-02T09:21:28,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/52f62d1dbfa442ca8f2ca65ac06dbcae is 1080, key is row0065/info:/1733131288158/Put/seqid=0 2024-12-02T09:21:28,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741853_1029 (size=12509) 2024-12-02T09:21:28,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741853_1029 (size=12509) 2024-12-02T09:21:28,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/52f62d1dbfa442ca8f2ca65ac06dbcae 2024-12-02T09:21:28,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/52f62d1dbfa442ca8f2ca65ac06dbcae as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/52f62d1dbfa442ca8f2ca65ac06dbcae 2024-12-02T09:21:28,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/52f62d1dbfa442ca8f2ca65ac06dbcae, entries=7, sequenceid=96, filesize=12.2 K 2024-12-02T09:21:28,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for 88bc4495114400d5a5b4dc77ae590483 in 24ms, sequenceid=96, compaction requested=false 2024-12-02T09:21:28,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88bc4495114400d5a5b4dc77ae590483: 2024-12-02T09:21:28,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] regionserver.HRegion(8855): Flush requested on 88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:28,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88bc4495114400d5a5b4dc77ae590483 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB 2024-12-02T09:21:28,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/55977d80299145c99553ef5ebe1d2885 is 1080, key is row0072/info:/1733131288171/Put/seqid=0 2024-12-02T09:21:28,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741854_1030 (size=17894) 2024-12-02T09:21:28,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741854_1030 (size=17894) 2024-12-02T09:21:28,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/55977d80299145c99553ef5ebe1d2885 2024-12-02T09:21:28,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/55977d80299145c99553ef5ebe1d2885 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/55977d80299145c99553ef5ebe1d2885 2024-12-02T09:21:28,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/55977d80299145c99553ef5ebe1d2885, entries=12, sequenceid=111, filesize=17.5 K 2024-12-02T09:21:28,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=10.51 KB/10760 for 88bc4495114400d5a5b4dc77ae590483 in 21ms, sequenceid=111, compaction requested=true 2024-12-02T09:21:28,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88bc4495114400d5a5b4dc77ae590483: 2024-12-02T09:21:28,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88bc4495114400d5a5b4dc77ae590483:info, priority=-2147483648, current under compaction store size is 1 2024-12-02T09:21:28,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:21:28,216 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T09:21:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] regionserver.HRegion(8855): Flush requested on 88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:28,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88bc4495114400d5a5b4dc77ae590483 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB 2024-12-02T09:21:28,217 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38663 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T09:21:28,217 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1541): 88bc4495114400d5a5b4dc77ae590483/info is initiating minor compaction (all files) 2024-12-02T09:21:28,217 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88bc4495114400d5a5b4dc77ae590483/info in TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483. 2024-12-02T09:21:28,217 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/c3cc2ae7234040bca23de967853e6efd, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/52f62d1dbfa442ca8f2ca65ac06dbcae, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/55977d80299145c99553ef5ebe1d2885] into tmpdir=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp, totalSize=37.8 K 2024-12-02T09:21:28,218 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3cc2ae7234040bca23de967853e6efd, keycount=3, bloomtype=ROW, size=8.1 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733131266017 2024-12-02T09:21:28,218 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52f62d1dbfa442ca8f2ca65ac06dbcae, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733131288158 2024-12-02T09:21:28,219 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55977d80299145c99553ef5ebe1d2885, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733131288171 2024-12-02T09:21:28,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/bd31b575b3c24aa3b39f041189ff60e0 is 1080, key is row0084/info:/1733131288196/Put/seqid=0 2024-12-02T09:21:28,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741855_1031 (size=16817) 2024-12-02T09:21:28,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741855_1031 (size=16817) 2024-12-02T09:21:28,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/bd31b575b3c24aa3b39f041189ff60e0 2024-12-02T09:21:28,231 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88bc4495114400d5a5b4dc77ae590483#info#compaction#70 average throughput is 22.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T09:21:28,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/bd31b575b3c24aa3b39f041189ff60e0 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/bd31b575b3c24aa3b39f041189ff60e0 2024-12-02T09:21:28,231 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/39eaccd99c2c47d1a3bca7991b7602bd is 1080, key is row0062/info:/1733131266017/Put/seqid=0 2024-12-02T09:21:28,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741856_1032 (size=28855) 2024-12-02T09:21:28,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741856_1032 (size=28855) 2024-12-02T09:21:28,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/bd31b575b3c24aa3b39f041189ff60e0, entries=11, sequenceid=125, filesize=16.4 K 2024-12-02T09:21:28,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=2.10 KB/2152 for 88bc4495114400d5a5b4dc77ae590483 in 21ms, sequenceid=125, compaction requested=false 2024-12-02T09:21:28,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88bc4495114400d5a5b4dc77ae590483: 2024-12-02T09:21:28,242 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/39eaccd99c2c47d1a3bca7991b7602bd as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/39eaccd99c2c47d1a3bca7991b7602bd 2024-12-02T09:21:28,248 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88bc4495114400d5a5b4dc77ae590483/info of 88bc4495114400d5a5b4dc77ae590483 into 39eaccd99c2c47d1a3bca7991b7602bd(size=28.2 K), total size for store is 44.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T09:21:28,248 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88bc4495114400d5a5b4dc77ae590483: 2024-12-02T09:21:28,248 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483., storeName=88bc4495114400d5a5b4dc77ae590483/info, priority=13, startTime=1733131288216; duration=0sec 2024-12-02T09:21:28,248 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:21:28,248 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88bc4495114400d5a5b4dc77ae590483:info 2024-12-02T09:21:28,473 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:28,473 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:29,474 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:29,474 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:30,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] regionserver.HRegion(8855): Flush requested on 88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:30,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88bc4495114400d5a5b4dc77ae590483 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-02T09:21:30,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/1c9ebec28a504d359db8181a79f4dae3 is 1080, key is row0095/info:/1733131288218/Put/seqid=0 2024-12-02T09:21:30,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741857_1033 (size=12515) 2024-12-02T09:21:30,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741857_1033 (size=12515) 2024-12-02T09:21:30,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/1c9ebec28a504d359db8181a79f4dae3 2024-12-02T09:21:30,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/1c9ebec28a504d359db8181a79f4dae3 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/1c9ebec28a504d359db8181a79f4dae3 2024-12-02T09:21:30,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/1c9ebec28a504d359db8181a79f4dae3, entries=7, sequenceid=136, filesize=12.2 K 2024-12-02T09:21:30,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for 88bc4495114400d5a5b4dc77ae590483 in 24ms, sequenceid=136, compaction requested=true 2024-12-02T09:21:30,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88bc4495114400d5a5b4dc77ae590483: 2024-12-02T09:21:30,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88bc4495114400d5a5b4dc77ae590483:info, priority=-2147483648, current under compaction store size is 1 2024-12-02T09:21:30,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:21:30,253 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T09:21:30,254 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 58187 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T09:21:30,254 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1541): 88bc4495114400d5a5b4dc77ae590483/info is initiating minor compaction (all files) 2024-12-02T09:21:30,254 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88bc4495114400d5a5b4dc77ae590483/info in TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483. 2024-12-02T09:21:30,255 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/39eaccd99c2c47d1a3bca7991b7602bd, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/bd31b575b3c24aa3b39f041189ff60e0, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/1c9ebec28a504d359db8181a79f4dae3] into tmpdir=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp, totalSize=56.8 K 2024-12-02T09:21:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] regionserver.HRegion(8855): Flush requested on 88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:30,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88bc4495114400d5a5b4dc77ae590483 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB 2024-12-02T09:21:30,255 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39eaccd99c2c47d1a3bca7991b7602bd, keycount=22, bloomtype=ROW, size=28.2 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1733131266017 2024-12-02T09:21:30,255 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd31b575b3c24aa3b39f041189ff60e0, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1733131288196 2024-12-02T09:21:30,256 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c9ebec28a504d359db8181a79f4dae3, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733131288218 2024-12-02T09:21:30,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/274eb14dd4674dd4adc874fddce7d0a1 is 1080, key is row0102/info:/1733131290230/Put/seqid=0 2024-12-02T09:21:30,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741858_1034 (size=17906) 2024-12-02T09:21:30,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741858_1034 (size=17906) 2024-12-02T09:21:30,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/274eb14dd4674dd4adc874fddce7d0a1 2024-12-02T09:21:30,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/274eb14dd4674dd4adc874fddce7d0a1 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/274eb14dd4674dd4adc874fddce7d0a1 2024-12-02T09:21:30,277 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88bc4495114400d5a5b4dc77ae590483#info#compaction#73 average throughput is 41.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T09:21:30,277 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/bb5dc492b9f742ff96744fc4f4e25127 is 1080, key is row0062/info:/1733131266017/Put/seqid=0 2024-12-02T09:21:30,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/274eb14dd4674dd4adc874fddce7d0a1, entries=12, sequenceid=151, filesize=17.5 K 2024-12-02T09:21:30,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=10.51 KB/10760 for 88bc4495114400d5a5b4dc77ae590483 in 27ms, sequenceid=151, compaction requested=false 2024-12-02T09:21:30,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88bc4495114400d5a5b4dc77ae590483: 2024-12-02T09:21:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] regionserver.HRegion(8855): Flush requested on 88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:30,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88bc4495114400d5a5b4dc77ae590483 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB 2024-12-02T09:21:30,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741859_1035 (size=48385) 2024-12-02T09:21:30,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/74393b454d3e4e1f8f7d5458242958c8 is 1080, key is row0114/info:/1733131290256/Put/seqid=0 2024-12-02T09:21:30,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741859_1035 (size=48385) 2024-12-02T09:21:30,297 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/bb5dc492b9f742ff96744fc4f4e25127 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/bb5dc492b9f742ff96744fc4f4e25127 2024-12-02T09:21:30,303 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88bc4495114400d5a5b4dc77ae590483/info of 88bc4495114400d5a5b4dc77ae590483 into bb5dc492b9f742ff96744fc4f4e25127(size=47.3 K), total size for store is 64.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T09:21:30,304 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88bc4495114400d5a5b4dc77ae590483: 2024-12-02T09:21:30,304 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483., storeName=88bc4495114400d5a5b4dc77ae590483/info, priority=13, startTime=1733131290253; duration=0sec 2024-12-02T09:21:30,304 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:21:30,304 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88bc4495114400d5a5b4dc77ae590483:info 2024-12-02T09:21:30,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741860_1036 (size=16828) 2024-12-02T09:21:30,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741860_1036 (size=16828) 2024-12-02T09:21:30,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/74393b454d3e4e1f8f7d5458242958c8 2024-12-02T09:21:30,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/74393b454d3e4e1f8f7d5458242958c8 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/74393b454d3e4e1f8f7d5458242958c8 2024-12-02T09:21:30,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/74393b454d3e4e1f8f7d5458242958c8, entries=11, sequenceid=165, filesize=16.4 K 2024-12-02T09:21:30,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=4.20 KB/4304 for 88bc4495114400d5a5b4dc77ae590483 in 33ms, sequenceid=165, compaction requested=true 2024-12-02T09:21:30,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88bc4495114400d5a5b4dc77ae590483: 2024-12-02T09:21:30,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88bc4495114400d5a5b4dc77ae590483:info, priority=-2147483648, current under compaction store size is 1 2024-12-02T09:21:30,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:21:30,316 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T09:21:30,317 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 83119 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T09:21:30,317 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1541): 88bc4495114400d5a5b4dc77ae590483/info is initiating minor compaction (all files) 2024-12-02T09:21:30,317 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88bc4495114400d5a5b4dc77ae590483/info in TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483. 2024-12-02T09:21:30,317 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/bb5dc492b9f742ff96744fc4f4e25127, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/274eb14dd4674dd4adc874fddce7d0a1, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/74393b454d3e4e1f8f7d5458242958c8] into tmpdir=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp, totalSize=81.2 K 2024-12-02T09:21:30,318 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb5dc492b9f742ff96744fc4f4e25127, keycount=40, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1733131266017 2024-12-02T09:21:30,318 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting 274eb14dd4674dd4adc874fddce7d0a1, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733131290230 2024-12-02T09:21:30,318 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74393b454d3e4e1f8f7d5458242958c8, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733131290256 2024-12-02T09:21:30,329 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88bc4495114400d5a5b4dc77ae590483#info#compaction#75 average throughput is 64.65 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T09:21:30,330 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/01e66c79b8ba490c9b53fe3c162a69ac is 1080, key is row0062/info:/1733131266017/Put/seqid=0 2024-12-02T09:21:30,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741861_1037 (size=73422) 2024-12-02T09:21:30,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741861_1037 (size=73422) 2024-12-02T09:21:30,347 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/01e66c79b8ba490c9b53fe3c162a69ac as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/01e66c79b8ba490c9b53fe3c162a69ac 2024-12-02T09:21:30,353 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88bc4495114400d5a5b4dc77ae590483/info of 88bc4495114400d5a5b4dc77ae590483 into 01e66c79b8ba490c9b53fe3c162a69ac(size=71.7 K), total size for store is 71.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T09:21:30,353 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88bc4495114400d5a5b4dc77ae590483: 2024-12-02T09:21:30,353 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483., storeName=88bc4495114400d5a5b4dc77ae590483/info, priority=13, startTime=1733131290316; duration=0sec 2024-12-02T09:21:30,353 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:21:30,353 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88bc4495114400d5a5b4dc77ae590483:info 2024-12-02T09:21:30,474 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:30,474 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:31,475 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:31,475 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:32,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] regionserver.HRegion(8855): Flush requested on 88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:32,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88bc4495114400d5a5b4dc77ae590483 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-02T09:21:32,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/f5815276a0f646ddad2478ed45e19072 is 1080, key is row0125/info:/1733131290284/Put/seqid=0 2024-12-02T09:21:32,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741862_1038 (size=12516) 2024-12-02T09:21:32,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741862_1038 (size=12516) 2024-12-02T09:21:32,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/f5815276a0f646ddad2478ed45e19072 2024-12-02T09:21:32,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/f5815276a0f646ddad2478ed45e19072 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/f5815276a0f646ddad2478ed45e19072 2024-12-02T09:21:32,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/f5815276a0f646ddad2478ed45e19072, entries=7, sequenceid=177, filesize=12.2 K 2024-12-02T09:21:32,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for 88bc4495114400d5a5b4dc77ae590483 in 24ms, sequenceid=177, compaction requested=false 2024-12-02T09:21:32,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88bc4495114400d5a5b4dc77ae590483: 2024-12-02T09:21:32,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] regionserver.HRegion(8855): Flush requested on 88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:32,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88bc4495114400d5a5b4dc77ae590483 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB 2024-12-02T09:21:32,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/deddb25941a24822a72c29af5d366841 is 1080, key is row0132/info:/1733131292299/Put/seqid=0 2024-12-02T09:21:32,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741863_1039 (size=17906) 2024-12-02T09:21:32,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741863_1039 (size=17906) 2024-12-02T09:21:32,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/deddb25941a24822a72c29af5d366841 2024-12-02T09:21:32,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/deddb25941a24822a72c29af5d366841 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/deddb25941a24822a72c29af5d366841 2024-12-02T09:21:32,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/deddb25941a24822a72c29af5d366841, entries=12, sequenceid=192, filesize=17.5 K 2024-12-02T09:21:32,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=9.46 KB/9684 for 88bc4495114400d5a5b4dc77ae590483 in 42ms, sequenceid=192, compaction requested=true 2024-12-02T09:21:32,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88bc4495114400d5a5b4dc77ae590483: 2024-12-02T09:21:32,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88bc4495114400d5a5b4dc77ae590483:info, priority=-2147483648, current under compaction store size is 1 2024-12-02T09:21:32,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:21:32,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] regionserver.HRegion(8855): Flush requested on 88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:32,367 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T09:21:32,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88bc4495114400d5a5b4dc77ae590483 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB 2024-12-02T09:21:32,368 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103844 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T09:21:32,368 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1541): 88bc4495114400d5a5b4dc77ae590483/info is initiating minor compaction (all files) 2024-12-02T09:21:32,368 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88bc4495114400d5a5b4dc77ae590483/info in TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483. 2024-12-02T09:21:32,368 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/01e66c79b8ba490c9b53fe3c162a69ac, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/f5815276a0f646ddad2478ed45e19072, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/deddb25941a24822a72c29af5d366841] into tmpdir=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp, totalSize=101.4 K 2024-12-02T09:21:32,369 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01e66c79b8ba490c9b53fe3c162a69ac, keycount=63, bloomtype=ROW, size=71.7 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733131266017 2024-12-02T09:21:32,369 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5815276a0f646ddad2478ed45e19072, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733131290284 2024-12-02T09:21:32,370 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting deddb25941a24822a72c29af5d366841, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733131292299 2024-12-02T09:21:32,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/87efa17457be459ebdc09c8d558a79dc is 1080, key is row0144/info:/1733131292325/Put/seqid=0 2024-12-02T09:21:32,390 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88bc4495114400d5a5b4dc77ae590483#info#compaction#79 average throughput is 28.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T09:21:32,391 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/4fee57cf938349cf966ea8aa4bdebb0c is 1080, key is row0062/info:/1733131266017/Put/seqid=0 2024-12-02T09:21:32,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741864_1040 (size=15750) 2024-12-02T09:21:32,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741864_1040 (size=15750) 2024-12-02T09:21:32,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/87efa17457be459ebdc09c8d558a79dc 2024-12-02T09:21:32,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741865_1041 (size=94010) 2024-12-02T09:21:32,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741865_1041 (size=94010) 2024-12-02T09:21:32,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/87efa17457be459ebdc09c8d558a79dc as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/87efa17457be459ebdc09c8d558a79dc 2024-12-02T09:21:32,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/87efa17457be459ebdc09c8d558a79dc, entries=10, sequenceid=205, filesize=15.4 K 2024-12-02T09:21:32,409 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/4fee57cf938349cf966ea8aa4bdebb0c as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/4fee57cf938349cf966ea8aa4bdebb0c 2024-12-02T09:21:32,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=7.36 KB/7532 for 88bc4495114400d5a5b4dc77ae590483 in 42ms, sequenceid=205, compaction requested=false 2024-12-02T09:21:32,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88bc4495114400d5a5b4dc77ae590483: 2024-12-02T09:21:32,415 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88bc4495114400d5a5b4dc77ae590483/info of 88bc4495114400d5a5b4dc77ae590483 into 4fee57cf938349cf966ea8aa4bdebb0c(size=91.8 K), total size for store is 107.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T09:21:32,415 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88bc4495114400d5a5b4dc77ae590483: 2024-12-02T09:21:32,415 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483., storeName=88bc4495114400d5a5b4dc77ae590483/info, priority=13, startTime=1733131292366; duration=0sec 2024-12-02T09:21:32,415 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:21:32,415 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88bc4495114400d5a5b4dc77ae590483:info 2024-12-02T09:21:32,475 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:32,475 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:33,476 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:33,476 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:33,867 INFO [master/7c6d666a4939:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-02T09:21:33,867 INFO [master/7c6d666a4939:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-02T09:21:34,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] regionserver.HRegion(8855): Flush requested on 88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:34,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88bc4495114400d5a5b4dc77ae590483 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB 2024-12-02T09:21:34,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/1506f530ed67453ea8f6c2b37d274eed is 1080, key is row0154/info:/1733131292369/Put/seqid=0 2024-12-02T09:21:34,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741866_1042 (size=13594) 2024-12-02T09:21:34,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741866_1042 (size=13594) 2024-12-02T09:21:34,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/1506f530ed67453ea8f6c2b37d274eed 2024-12-02T09:21:34,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/1506f530ed67453ea8f6c2b37d274eed as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/1506f530ed67453ea8f6c2b37d274eed 2024-12-02T09:21:34,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/1506f530ed67453ea8f6c2b37d274eed, entries=8, sequenceid=217, filesize=13.3 K 2024-12-02T09:21:34,420 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=10.51 KB/10760 for 88bc4495114400d5a5b4dc77ae590483 in 33ms, sequenceid=217, compaction requested=true 2024-12-02T09:21:34,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88bc4495114400d5a5b4dc77ae590483: 2024-12-02T09:21:34,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88bc4495114400d5a5b4dc77ae590483:info, priority=-2147483648, current under compaction store size is 1 2024-12-02T09:21:34,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:21:34,420 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T09:21:34,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] regionserver.HRegion(8855): Flush requested on 88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:34,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88bc4495114400d5a5b4dc77ae590483 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB 2024-12-02T09:21:34,421 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 123354 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T09:21:34,422 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1541): 88bc4495114400d5a5b4dc77ae590483/info is initiating minor compaction (all files) 2024-12-02T09:21:34,422 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88bc4495114400d5a5b4dc77ae590483/info in TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483. 2024-12-02T09:21:34,422 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/4fee57cf938349cf966ea8aa4bdebb0c, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/87efa17457be459ebdc09c8d558a79dc, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/1506f530ed67453ea8f6c2b37d274eed] into tmpdir=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp, totalSize=120.5 K 2024-12-02T09:21:34,422 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fee57cf938349cf966ea8aa4bdebb0c, keycount=82, bloomtype=ROW, size=91.8 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733131266017 2024-12-02T09:21:34,423 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87efa17457be459ebdc09c8d558a79dc, keycount=10, bloomtype=ROW, size=15.4 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733131292325 2024-12-02T09:21:34,423 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1506f530ed67453ea8f6c2b37d274eed, keycount=8, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733131292369 2024-12-02T09:21:34,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/c5611af52ec249d98537199814ab72f9 is 1080, key is row0162/info:/1733131294389/Put/seqid=0 2024-12-02T09:21:34,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741867_1043 (size=16828) 2024-12-02T09:21:34,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741867_1043 (size=16828) 2024-12-02T09:21:34,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/c5611af52ec249d98537199814ab72f9 2024-12-02T09:21:34,445 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88bc4495114400d5a5b4dc77ae590483#info#compaction#82 average throughput is 34.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T09:21:34,446 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/8af0a4a4a41d48e8926fe48fc4884419 is 1080, key is row0062/info:/1733131266017/Put/seqid=0 2024-12-02T09:21:34,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/c5611af52ec249d98537199814ab72f9 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/c5611af52ec249d98537199814ab72f9 2024-12-02T09:21:34,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/c5611af52ec249d98537199814ab72f9, entries=11, sequenceid=231, filesize=16.4 K 2024-12-02T09:21:34,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=16.81 KB/17216 for 88bc4495114400d5a5b4dc77ae590483 in 34ms, sequenceid=231, compaction requested=false 2024-12-02T09:21:34,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88bc4495114400d5a5b4dc77ae590483: 2024-12-02T09:21:34,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] regionserver.HRegion(8855): Flush requested on 88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:34,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88bc4495114400d5a5b4dc77ae590483 1/1 column families, dataSize=17.86 KB heapSize=19.38 KB 2024-12-02T09:21:34,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/65e7a15a6382408db185ba7220dcffc2 is 1080, key is row0173/info:/1733131294422/Put/seqid=0 2024-12-02T09:21:34,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741868_1044 (size=113520) 2024-12-02T09:21:34,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741868_1044 (size=113520) 2024-12-02T09:21:34,474 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/8af0a4a4a41d48e8926fe48fc4884419 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/8af0a4a4a41d48e8926fe48fc4884419 2024-12-02T09:21:34,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741869_1045 (size=23316) 2024-12-02T09:21:34,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741869_1045 (size=23316) 2024-12-02T09:21:34,477 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:34,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.86 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/65e7a15a6382408db185ba7220dcffc2 2024-12-02T09:21:34,479 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:34,481 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88bc4495114400d5a5b4dc77ae590483/info of 88bc4495114400d5a5b4dc77ae590483 into 8af0a4a4a41d48e8926fe48fc4884419(size=110.9 K), total size for store is 127.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T09:21:34,481 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88bc4495114400d5a5b4dc77ae590483: 2024-12-02T09:21:34,481 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483., storeName=88bc4495114400d5a5b4dc77ae590483/info, priority=13, startTime=1733131294420; duration=0sec 2024-12-02T09:21:34,481 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:21:34,481 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88bc4495114400d5a5b4dc77ae590483:info 2024-12-02T09:21:34,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/65e7a15a6382408db185ba7220dcffc2 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/65e7a15a6382408db185ba7220dcffc2 2024-12-02T09:21:34,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/65e7a15a6382408db185ba7220dcffc2, entries=17, sequenceid=251, filesize=22.8 K 2024-12-02T09:21:34,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~17.86 KB/18292, heapSize ~19.36 KB/19824, currentSize=3.15 KB/3228 for 88bc4495114400d5a5b4dc77ae590483 in 33ms, sequenceid=251, compaction requested=true 2024-12-02T09:21:34,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88bc4495114400d5a5b4dc77ae590483: 2024-12-02T09:21:34,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88bc4495114400d5a5b4dc77ae590483:info, priority=-2147483648, current under compaction store size is 1 2024-12-02T09:21:34,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:21:34,489 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T09:21:34,490 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 153664 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T09:21:34,490 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1541): 88bc4495114400d5a5b4dc77ae590483/info is initiating minor compaction (all files) 2024-12-02T09:21:34,491 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88bc4495114400d5a5b4dc77ae590483/info in TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483. 2024-12-02T09:21:34,491 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/8af0a4a4a41d48e8926fe48fc4884419, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/c5611af52ec249d98537199814ab72f9, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/65e7a15a6382408db185ba7220dcffc2] into tmpdir=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp, totalSize=150.1 K 2024-12-02T09:21:34,491 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8af0a4a4a41d48e8926fe48fc4884419, keycount=100, bloomtype=ROW, size=110.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733131266017 2024-12-02T09:21:34,491 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5611af52ec249d98537199814ab72f9, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733131294389 2024-12-02T09:21:34,492 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65e7a15a6382408db185ba7220dcffc2, keycount=17, bloomtype=ROW, size=22.8 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733131294422 2024-12-02T09:21:34,503 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88bc4495114400d5a5b4dc77ae590483#info#compaction#84 average throughput is 43.78 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T09:21:34,503 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/56a32993656a4ba5b527e4125e5a351a is 1080, key is row0062/info:/1733131266017/Put/seqid=0 2024-12-02T09:21:34,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741870_1046 (size=144011) 2024-12-02T09:21:34,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741870_1046 (size=144011) 2024-12-02T09:21:34,512 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/56a32993656a4ba5b527e4125e5a351a as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/56a32993656a4ba5b527e4125e5a351a 2024-12-02T09:21:34,518 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88bc4495114400d5a5b4dc77ae590483/info of 88bc4495114400d5a5b4dc77ae590483 into 56a32993656a4ba5b527e4125e5a351a(size=140.6 K), total size for store is 140.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T09:21:34,518 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88bc4495114400d5a5b4dc77ae590483: 2024-12-02T09:21:34,518 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483., storeName=88bc4495114400d5a5b4dc77ae590483/info, priority=13, startTime=1733131294489; duration=0sec 2024-12-02T09:21:34,519 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:21:34,519 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88bc4495114400d5a5b4dc77ae590483:info 2024-12-02T09:21:35,477 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:35,479 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] regionserver.HRegion(8855): Flush requested on 88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:36,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88bc4495114400d5a5b4dc77ae590483 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-02T09:21:36,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/007b44ab25a04b439d7923bc07048a7e is 1080, key is row0190/info:/1733131294457/Put/seqid=0 2024-12-02T09:21:36,478 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:36,480 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:36,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741871_1047 (size=12520) 2024-12-02T09:21:36,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741871_1047 (size=12520) 2024-12-02T09:21:36,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/007b44ab25a04b439d7923bc07048a7e 2024-12-02T09:21:36,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/007b44ab25a04b439d7923bc07048a7e as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/007b44ab25a04b439d7923bc07048a7e 2024-12-02T09:21:36,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/007b44ab25a04b439d7923bc07048a7e, entries=7, sequenceid=263, filesize=12.2 K 2024-12-02T09:21:36,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=12.61 KB/12912 for 88bc4495114400d5a5b4dc77ae590483 in 24ms, sequenceid=263, compaction requested=false 2024-12-02T09:21:36,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88bc4495114400d5a5b4dc77ae590483: 2024-12-02T09:21:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] regionserver.HRegion(8855): Flush requested on 88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:36,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88bc4495114400d5a5b4dc77ae590483 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB 2024-12-02T09:21:36,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/23ce14b9b6d94daf93facfe3a14c62c3 is 1080, key is row0197/info:/1733131296471/Put/seqid=0 2024-12-02T09:21:36,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741872_1048 (size=19013) 2024-12-02T09:21:36,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741872_1048 (size=19013) 2024-12-02T09:21:36,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/23ce14b9b6d94daf93facfe3a14c62c3 2024-12-02T09:21:36,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/23ce14b9b6d94daf93facfe3a14c62c3 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/23ce14b9b6d94daf93facfe3a14c62c3 2024-12-02T09:21:36,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/23ce14b9b6d94daf93facfe3a14c62c3, entries=13, sequenceid=279, filesize=18.6 K 2024-12-02T09:21:36,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=11.56 KB/11836 for 88bc4495114400d5a5b4dc77ae590483 in 20ms, sequenceid=279, compaction requested=true 2024-12-02T09:21:36,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88bc4495114400d5a5b4dc77ae590483: 2024-12-02T09:21:36,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88bc4495114400d5a5b4dc77ae590483:info, priority=-2147483648, current under compaction store size is 1 2024-12-02T09:21:36,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:21:36,515 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T09:21:36,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] regionserver.HRegion(8855): Flush requested on 88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:36,516 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 175544 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T09:21:36,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88bc4495114400d5a5b4dc77ae590483 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB 2024-12-02T09:21:36,516 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1541): 88bc4495114400d5a5b4dc77ae590483/info is initiating minor compaction (all files) 2024-12-02T09:21:36,516 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88bc4495114400d5a5b4dc77ae590483/info in TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483. 2024-12-02T09:21:36,516 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/56a32993656a4ba5b527e4125e5a351a, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/007b44ab25a04b439d7923bc07048a7e, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/23ce14b9b6d94daf93facfe3a14c62c3] into tmpdir=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp, totalSize=171.4 K 2024-12-02T09:21:36,516 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56a32993656a4ba5b527e4125e5a351a, keycount=128, bloomtype=ROW, size=140.6 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733131266017 2024-12-02T09:21:36,517 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting 007b44ab25a04b439d7923bc07048a7e, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733131294457 2024-12-02T09:21:36,517 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23ce14b9b6d94daf93facfe3a14c62c3, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733131296471 2024-12-02T09:21:36,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/81158fa14bf94fe48bc47653f8dc50b4 is 1080, key is row0210/info:/1733131296495/Put/seqid=0 2024-12-02T09:21:36,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741873_1049 (size=17918) 2024-12-02T09:21:36,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741873_1049 (size=17918) 2024-12-02T09:21:36,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/81158fa14bf94fe48bc47653f8dc50b4 2024-12-02T09:21:36,530 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88bc4495114400d5a5b4dc77ae590483#info#compaction#88 average throughput is 50.62 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T09:21:36,530 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/ec7548350a5d4dfc8bbfb3dca3febbea is 1080, key is row0062/info:/1733131266017/Put/seqid=0 2024-12-02T09:21:36,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/81158fa14bf94fe48bc47653f8dc50b4 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/81158fa14bf94fe48bc47653f8dc50b4 2024-12-02T09:21:36,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741874_1050 (size=165694) 2024-12-02T09:21:36,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741874_1050 (size=165694) 2024-12-02T09:21:36,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/81158fa14bf94fe48bc47653f8dc50b4, entries=12, sequenceid=294, filesize=17.5 K 2024-12-02T09:21:36,540 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/ec7548350a5d4dfc8bbfb3dca3febbea as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/ec7548350a5d4dfc8bbfb3dca3febbea 2024-12-02T09:21:36,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=3.15 KB/3228 for 88bc4495114400d5a5b4dc77ae590483 in 24ms, sequenceid=294, compaction requested=false 2024-12-02T09:21:36,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88bc4495114400d5a5b4dc77ae590483: 2024-12-02T09:21:36,546 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88bc4495114400d5a5b4dc77ae590483/info of 88bc4495114400d5a5b4dc77ae590483 into ec7548350a5d4dfc8bbfb3dca3febbea(size=161.8 K), total size for store is 179.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T09:21:36,546 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88bc4495114400d5a5b4dc77ae590483: 2024-12-02T09:21:36,546 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483., storeName=88bc4495114400d5a5b4dc77ae590483/info, priority=13, startTime=1733131296515; duration=0sec 2024-12-02T09:21:36,546 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:21:36,546 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88bc4495114400d5a5b4dc77ae590483:info 2024-12-02T09:21:37,478 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:37,480 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:38,479 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:38,481 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:38,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] regionserver.HRegion(8855): Flush requested on 88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:38,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88bc4495114400d5a5b4dc77ae590483 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-12-02T09:21:38,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/949fd0e890f244ef9f9a4cf4bec7f6cd is 1080, key is row0222/info:/1733131296517/Put/seqid=0 2024-12-02T09:21:38,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741875_1051 (size=12523) 2024-12-02T09:21:38,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741875_1051 (size=12523) 2024-12-02T09:21:38,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/949fd0e890f244ef9f9a4cf4bec7f6cd 2024-12-02T09:21:38,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/949fd0e890f244ef9f9a4cf4bec7f6cd as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/949fd0e890f244ef9f9a4cf4bec7f6cd 2024-12-02T09:21:38,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/949fd0e890f244ef9f9a4cf4bec7f6cd, entries=7, sequenceid=305, filesize=12.2 K 2024-12-02T09:21:38,558 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for 88bc4495114400d5a5b4dc77ae590483 in 21ms, sequenceid=305, compaction requested=true 2024-12-02T09:21:38,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88bc4495114400d5a5b4dc77ae590483: 2024-12-02T09:21:38,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88bc4495114400d5a5b4dc77ae590483:info, priority=-2147483648, current under compaction store size is 1 2024-12-02T09:21:38,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:21:38,558 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-02T09:21:38,559 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 196135 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-02T09:21:38,560 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1541): 88bc4495114400d5a5b4dc77ae590483/info is initiating minor compaction (all files) 2024-12-02T09:21:38,560 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88bc4495114400d5a5b4dc77ae590483/info in TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483. 2024-12-02T09:21:38,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] regionserver.HRegion(8855): Flush requested on 88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:38,560 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/ec7548350a5d4dfc8bbfb3dca3febbea, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/81158fa14bf94fe48bc47653f8dc50b4, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/949fd0e890f244ef9f9a4cf4bec7f6cd] into tmpdir=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp, totalSize=191.5 K 2024-12-02T09:21:38,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88bc4495114400d5a5b4dc77ae590483 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB 2024-12-02T09:21:38,560 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec7548350a5d4dfc8bbfb3dca3febbea, keycount=148, bloomtype=ROW, size=161.8 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733131266017 2024-12-02T09:21:38,560 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81158fa14bf94fe48bc47653f8dc50b4, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733131296495 2024-12-02T09:21:38,561 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting 949fd0e890f244ef9f9a4cf4bec7f6cd, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733131296517 2024-12-02T09:21:38,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/de8f1479fbea4fd09d81f5f81e3097ee is 1080, key is row0229/info:/1733131298538/Put/seqid=0 2024-12-02T09:21:38,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741876_1052 (size=19013) 2024-12-02T09:21:38,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741876_1052 (size=19013) 2024-12-02T09:21:38,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/de8f1479fbea4fd09d81f5f81e3097ee 2024-12-02T09:21:38,576 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88bc4495114400d5a5b4dc77ae590483#info#compaction#91 average throughput is 42.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-02T09:21:38,577 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/1e42b5d0b0ee483686cc078f2ce19f82 is 1080, key is row0062/info:/1733131266017/Put/seqid=0 2024-12-02T09:21:38,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/de8f1479fbea4fd09d81f5f81e3097ee as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/de8f1479fbea4fd09d81f5f81e3097ee 2024-12-02T09:21:38,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/de8f1479fbea4fd09d81f5f81e3097ee, entries=13, sequenceid=321, filesize=18.6 K 2024-12-02T09:21:38,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=15.76 KB/16140 for 88bc4495114400d5a5b4dc77ae590483 in 31ms, sequenceid=321, compaction requested=false 2024-12-02T09:21:38,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88bc4495114400d5a5b4dc77ae590483: 2024-12-02T09:21:38,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741877_1053 (size=186301) 2024-12-02T09:21:38,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741877_1053 (size=186301) 2024-12-02T09:21:38,604 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/1e42b5d0b0ee483686cc078f2ce19f82 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/1e42b5d0b0ee483686cc078f2ce19f82 2024-12-02T09:21:38,610 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88bc4495114400d5a5b4dc77ae590483/info of 88bc4495114400d5a5b4dc77ae590483 into 1e42b5d0b0ee483686cc078f2ce19f82(size=181.9 K), total size for store is 200.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-02T09:21:38,610 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88bc4495114400d5a5b4dc77ae590483: 2024-12-02T09:21:38,610 INFO [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483., storeName=88bc4495114400d5a5b4dc77ae590483/info, priority=13, startTime=1733131298558; duration=0sec 2024-12-02T09:21:38,610 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-02T09:21:38,610 DEBUG [RS:0;7c6d666a4939:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88bc4495114400d5a5b4dc77ae590483:info 2024-12-02T09:21:38,686 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 20340 2024-12-02T09:21:39,479 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:39,481 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:40,480 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:40,481 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:40,591 INFO [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files 2024-12-02T09:21:40,592 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C37787%2C1733131252687.1733131300592 2024-12-02T09:21:40,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:40,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:40,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:40,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:40,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:40,602 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/WALs/7c6d666a4939,37787,1733131252687/7c6d666a4939%2C37787%2C1733131252687.1733131253294 with entries=316, filesize=309.49 KB; new WAL /user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/WALs/7c6d666a4939,37787,1733131252687/7c6d666a4939%2C37787%2C1733131252687.1733131300592 2024-12-02T09:21:40,603 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45143:45143),(127.0.0.1/127.0.0.1:46257:46257)] 2024-12-02T09:21:40,604 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/WALs/7c6d666a4939,37787,1733131252687/7c6d666a4939%2C37787%2C1733131252687.1733131253294 is not closed yet, will try archiving it next time 2024-12-02T09:21:40,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741833_1009 (size=316924) 2024-12-02T09:21:40,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741833_1009 (size=316924) 2024-12-02T09:21:40,609 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 88bc4495114400d5a5b4dc77ae590483 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB 2024-12-02T09:21:40,613 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/a5020c6ba9494ecea1da6c8790e4b922 is 1080, key is row0242/info:/1733131298561/Put/seqid=0 2024-12-02T09:21:40,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741879_1055 (size=21171) 2024-12-02T09:21:40,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741879_1055 (size=21171) 2024-12-02T09:21:41,018 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/a5020c6ba9494ecea1da6c8790e4b922 2024-12-02T09:21:41,023 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/.tmp/info/a5020c6ba9494ecea1da6c8790e4b922 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/a5020c6ba9494ecea1da6c8790e4b922 2024-12-02T09:21:41,027 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/a5020c6ba9494ecea1da6c8790e4b922, entries=15, sequenceid=340, filesize=20.7 K 2024-12-02T09:21:41,028 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=0 B/0 for 88bc4495114400d5a5b4dc77ae590483 in 419ms, sequenceid=340, compaction requested=true 2024-12-02T09:21:41,028 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 88bc4495114400d5a5b4dc77ae590483: 2024-12-02T09:21:41,029 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=705 B heapSize=2.05 KB 2024-12-02T09:21:41,033 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/hbase/meta/1588230740/.tmp/info/57bc0326ad0a43758e153750676b1167 is 193, key is TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483./info:regioninfo/1733131276888/Put/seqid=0 2024-12-02T09:21:41,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741880_1056 (size=6223) 2024-12-02T09:21:41,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741880_1056 (size=6223) 2024-12-02T09:21:41,037 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=705 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/hbase/meta/1588230740/.tmp/info/57bc0326ad0a43758e153750676b1167 2024-12-02T09:21:41,042 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/hbase/meta/1588230740/.tmp/info/57bc0326ad0a43758e153750676b1167 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/hbase/meta/1588230740/info/57bc0326ad0a43758e153750676b1167 2024-12-02T09:21:41,046 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/hbase/meta/1588230740/info/57bc0326ad0a43758e153750676b1167, entries=5, sequenceid=21, filesize=6.1 K 2024-12-02T09:21:41,047 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~705 B/705, heapSize ~1.29 KB/1320, currentSize=0 B/0 for 1588230740 in 18ms, sequenceid=21, compaction requested=false 2024-12-02T09:21:41,047 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-12-02T09:21:41,047 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e2c45d5b11370f58eb0c1f4939fe5ffa: 2024-12-02T09:21:41,048 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C37787%2C1733131252687.1733131301048 2024-12-02T09:21:41,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:41,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:41,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:41,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:41,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:41,052 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/WALs/7c6d666a4939,37787,1733131252687/7c6d666a4939%2C37787%2C1733131252687.1733131300592 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/WALs/7c6d666a4939,37787,1733131252687/7c6d666a4939%2C37787%2C1733131252687.1733131301048 2024-12-02T09:21:41,053 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46257:46257),(127.0.0.1/127.0.0.1:45143:45143)] 2024-12-02T09:21:41,053 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/WALs/7c6d666a4939,37787,1733131252687/7c6d666a4939%2C37787%2C1733131252687.1733131300592 is not closed yet, will try archiving it next time 2024-12-02T09:21:41,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741878_1054 (size=731) 2024-12-02T09:21:41,053 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/WALs/7c6d666a4939,37787,1733131252687/7c6d666a4939%2C37787%2C1733131252687.1733131253294 to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/oldWALs/7c6d666a4939%2C37787%2C1733131252687.1733131253294 2024-12-02T09:21:41,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741878_1054 (size=731) 2024-12-02T09:21:41,054 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-12-02T09:21:41,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-02T09:21:41,054 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/WALs/7c6d666a4939,37787,1733131252687/7c6d666a4939%2C37787%2C1733131252687.1733131300592 to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/oldWALs/7c6d666a4939%2C37787%2C1733131252687.1733131300592 2024-12-02T09:21:41,054 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T09:21:41,054 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T09:21:41,054 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:21:41,055 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T09:21:41,055 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:21:41,055 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-02T09:21:41,055 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=400317172, stopped=false 2024-12-02T09:21:41,055 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7c6d666a4939,39695,1733131252558 2024-12-02T09:21:41,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T09:21:41,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1009a4a10d80001, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T09:21:41,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1009a4a10d80001, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:21:41,090 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T09:21:41,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:21:41,090 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T09:21:41,090 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T09:21:41,091 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:21:41,091 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:21:41,091 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37787-0x1009a4a10d80001, quorum=127.0.0.1:62506, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:21:41,091 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7c6d666a4939,37787,1733131252687' ***** 2024-12-02T09:21:41,091 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-02T09:21:41,092 INFO [RS:0;7c6d666a4939:37787 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T09:21:41,092 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T09:21:41,092 INFO [RS:0;7c6d666a4939:37787 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-02T09:21:41,092 INFO [RS:0;7c6d666a4939:37787 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-02T09:21:41,093 INFO [RS:0;7c6d666a4939:37787 {}] regionserver.HRegionServer(3091): Received CLOSE for 88bc4495114400d5a5b4dc77ae590483 2024-12-02T09:21:41,093 INFO [RS:0;7c6d666a4939:37787 {}] regionserver.HRegionServer(3091): Received CLOSE for e2c45d5b11370f58eb0c1f4939fe5ffa 2024-12-02T09:21:41,093 INFO [RS:0;7c6d666a4939:37787 {}] regionserver.HRegionServer(959): stopping server 7c6d666a4939,37787,1733131252687 2024-12-02T09:21:41,093 INFO [RS:0;7c6d666a4939:37787 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T09:21:41,093 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 88bc4495114400d5a5b4dc77ae590483, disabling compactions & flushes 2024-12-02T09:21:41,093 INFO [RS:0;7c6d666a4939:37787 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7c6d666a4939:37787. 2024-12-02T09:21:41,093 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483. 2024-12-02T09:21:41,093 DEBUG [RS:0;7c6d666a4939:37787 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T09:21:41,093 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483. 2024-12-02T09:21:41,093 DEBUG [RS:0;7c6d666a4939:37787 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:21:41,093 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483. after waiting 0 ms 2024-12-02T09:21:41,093 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483. 2024-12-02T09:21:41,093 INFO [RS:0;7c6d666a4939:37787 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T09:21:41,093 INFO [RS:0;7c6d666a4939:37787 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T09:21:41,093 INFO [RS:0;7c6d666a4939:37787 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T09:21:41,093 INFO [RS:0;7c6d666a4939:37787 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-02T09:21:41,094 INFO [RS:0;7c6d666a4939:37787 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-12-02T09:21:41,094 DEBUG [RS:0;7c6d666a4939:37787 {}] regionserver.HRegionServer(1325): Online Regions={88bc4495114400d5a5b4dc77ae590483=TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483., 1588230740=hbase:meta,,1.1588230740, e2c45d5b11370f58eb0c1f4939fe5ffa=TestLogRolling-testLogRolling,,1733131276213.e2c45d5b11370f58eb0c1f4939fe5ffa.} 2024-12-02T09:21:41,094 DEBUG [RS:0;7c6d666a4939:37787 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 88bc4495114400d5a5b4dc77ae590483, e2c45d5b11370f58eb0c1f4939fe5ffa 2024-12-02T09:21:41,094 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T09:21:41,094 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T09:21:41,094 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T09:21:41,094 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T09:21:41,094 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T09:21:41,094 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/bb27cad513c94baca1abc85930a76f31.adb545fc7773af344dd36d23ae47da66->hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/bb27cad513c94baca1abc85930a76f31-top, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/c3cc2ae7234040bca23de967853e6efd, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/52f62d1dbfa442ca8f2ca65ac06dbcae, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/39eaccd99c2c47d1a3bca7991b7602bd, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/55977d80299145c99553ef5ebe1d2885, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/bd31b575b3c24aa3b39f041189ff60e0, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/bb5dc492b9f742ff96744fc4f4e25127, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/1c9ebec28a504d359db8181a79f4dae3, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/274eb14dd4674dd4adc874fddce7d0a1, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/01e66c79b8ba490c9b53fe3c162a69ac, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/74393b454d3e4e1f8f7d5458242958c8, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/f5815276a0f646ddad2478ed45e19072, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/4fee57cf938349cf966ea8aa4bdebb0c, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/deddb25941a24822a72c29af5d366841, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/87efa17457be459ebdc09c8d558a79dc, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/8af0a4a4a41d48e8926fe48fc4884419, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/1506f530ed67453ea8f6c2b37d274eed, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/c5611af52ec249d98537199814ab72f9, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/56a32993656a4ba5b527e4125e5a351a, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/65e7a15a6382408db185ba7220dcffc2, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/007b44ab25a04b439d7923bc07048a7e, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/ec7548350a5d4dfc8bbfb3dca3febbea, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/23ce14b9b6d94daf93facfe3a14c62c3, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/81158fa14bf94fe48bc47653f8dc50b4, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/949fd0e890f244ef9f9a4cf4bec7f6cd] to archive 2024-12-02T09:21:41,096 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T09:21:41,097 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/bb27cad513c94baca1abc85930a76f31.adb545fc7773af344dd36d23ae47da66 to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/bb27cad513c94baca1abc85930a76f31.adb545fc7773af344dd36d23ae47da66 2024-12-02T09:21:41,099 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/c3cc2ae7234040bca23de967853e6efd to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/c3cc2ae7234040bca23de967853e6efd 2024-12-02T09:21:41,099 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-12-02T09:21:41,100 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T09:21:41,100 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T09:21:41,100 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733131301094Running coprocessor pre-close hooks at 1733131301094Disabling compacts and flushes for region at 1733131301094Disabling writes for close at 1733131301094Writing region close event to WAL at 1733131301096 (+2 ms)Running coprocessor post-close hooks at 1733131301100 (+4 ms)Closed at 1733131301100 2024-12-02T09:21:41,100 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-02T09:21:41,100 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/52f62d1dbfa442ca8f2ca65ac06dbcae to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/52f62d1dbfa442ca8f2ca65ac06dbcae 2024-12-02T09:21:41,102 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/39eaccd99c2c47d1a3bca7991b7602bd to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/39eaccd99c2c47d1a3bca7991b7602bd 2024-12-02T09:21:41,103 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/55977d80299145c99553ef5ebe1d2885 to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/55977d80299145c99553ef5ebe1d2885 2024-12-02T09:21:41,104 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/bd31b575b3c24aa3b39f041189ff60e0 to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/bd31b575b3c24aa3b39f041189ff60e0 2024-12-02T09:21:41,105 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/bb5dc492b9f742ff96744fc4f4e25127 to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/bb5dc492b9f742ff96744fc4f4e25127 2024-12-02T09:21:41,107 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/1c9ebec28a504d359db8181a79f4dae3 to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/1c9ebec28a504d359db8181a79f4dae3 2024-12-02T09:21:41,108 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/274eb14dd4674dd4adc874fddce7d0a1 to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/274eb14dd4674dd4adc874fddce7d0a1 2024-12-02T09:21:41,109 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/01e66c79b8ba490c9b53fe3c162a69ac to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/01e66c79b8ba490c9b53fe3c162a69ac 2024-12-02T09:21:41,109 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/74393b454d3e4e1f8f7d5458242958c8 to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/74393b454d3e4e1f8f7d5458242958c8 2024-12-02T09:21:41,110 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/f5815276a0f646ddad2478ed45e19072 to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/f5815276a0f646ddad2478ed45e19072 2024-12-02T09:21:41,111 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/4fee57cf938349cf966ea8aa4bdebb0c to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/4fee57cf938349cf966ea8aa4bdebb0c 2024-12-02T09:21:41,112 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/deddb25941a24822a72c29af5d366841 to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/deddb25941a24822a72c29af5d366841 2024-12-02T09:21:41,112 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/87efa17457be459ebdc09c8d558a79dc to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/87efa17457be459ebdc09c8d558a79dc 2024-12-02T09:21:41,113 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/8af0a4a4a41d48e8926fe48fc4884419 to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/8af0a4a4a41d48e8926fe48fc4884419 2024-12-02T09:21:41,114 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/1506f530ed67453ea8f6c2b37d274eed to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/1506f530ed67453ea8f6c2b37d274eed 2024-12-02T09:21:41,115 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/c5611af52ec249d98537199814ab72f9 to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/c5611af52ec249d98537199814ab72f9 2024-12-02T09:21:41,116 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/56a32993656a4ba5b527e4125e5a351a to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/56a32993656a4ba5b527e4125e5a351a 2024-12-02T09:21:41,117 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/65e7a15a6382408db185ba7220dcffc2 to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/65e7a15a6382408db185ba7220dcffc2 2024-12-02T09:21:41,118 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/007b44ab25a04b439d7923bc07048a7e to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/007b44ab25a04b439d7923bc07048a7e 2024-12-02T09:21:41,119 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/ec7548350a5d4dfc8bbfb3dca3febbea to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/ec7548350a5d4dfc8bbfb3dca3febbea 2024-12-02T09:21:41,120 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/23ce14b9b6d94daf93facfe3a14c62c3 to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/23ce14b9b6d94daf93facfe3a14c62c3 2024-12-02T09:21:41,121 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/81158fa14bf94fe48bc47653f8dc50b4 to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/81158fa14bf94fe48bc47653f8dc50b4 2024-12-02T09:21:41,122 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/949fd0e890f244ef9f9a4cf4bec7f6cd to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/info/949fd0e890f244ef9f9a4cf4bec7f6cd 2024-12-02T09:21:41,123 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=7c6d666a4939:39695 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-12-02T09:21:41,123 WARN [StoreCloser-TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [c3cc2ae7234040bca23de967853e6efd=8260, 52f62d1dbfa442ca8f2ca65ac06dbcae=12509, 39eaccd99c2c47d1a3bca7991b7602bd=28855, 55977d80299145c99553ef5ebe1d2885=17894, bd31b575b3c24aa3b39f041189ff60e0=16817, bb5dc492b9f742ff96744fc4f4e25127=48385, 1c9ebec28a504d359db8181a79f4dae3=12515, 274eb14dd4674dd4adc874fddce7d0a1=17906, 01e66c79b8ba490c9b53fe3c162a69ac=73422, 74393b454d3e4e1f8f7d5458242958c8=16828, f5815276a0f646ddad2478ed45e19072=12516, 4fee57cf938349cf966ea8aa4bdebb0c=94010, deddb25941a24822a72c29af5d366841=17906, 87efa17457be459ebdc09c8d558a79dc=15750, 8af0a4a4a41d48e8926fe48fc4884419=113520, 1506f530ed67453ea8f6c2b37d274eed=13594, c5611af52ec249d98537199814ab72f9=16828, 56a32993656a4ba5b527e4125e5a351a=144011, 65e7a15a6382408db185ba7220dcffc2=23316, 007b44ab25a04b439d7923bc07048a7e=12520, ec7548350a5d4dfc8bbfb3dca3febbea=165694, 23ce14b9b6d94daf93facfe3a14c62c3=19013, 81158fa14bf94fe48bc47653f8dc50b4=17918, 949fd0e890f244ef9f9a4cf4bec7f6cd=12523] 2024-12-02T09:21:41,126 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/88bc4495114400d5a5b4dc77ae590483/recovered.edits/343.seqid, newMaxSeqId=343, maxSeqId=85 2024-12-02T09:21:41,127 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483. 2024-12-02T09:21:41,127 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 88bc4495114400d5a5b4dc77ae590483: Waiting for close lock at 1733131301093Running coprocessor pre-close hooks at 1733131301093Disabling compacts and flushes for region at 1733131301093Disabling writes for close at 1733131301093Writing region close event to WAL at 1733131301123 (+30 ms)Running coprocessor post-close hooks at 1733131301127 (+4 ms)Closed at 1733131301127 2024-12-02T09:21:41,127 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1733131276213.88bc4495114400d5a5b4dc77ae590483. 2024-12-02T09:21:41,127 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e2c45d5b11370f58eb0c1f4939fe5ffa, disabling compactions & flushes 2024-12-02T09:21:41,127 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733131276213.e2c45d5b11370f58eb0c1f4939fe5ffa. 2024-12-02T09:21:41,127 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733131276213.e2c45d5b11370f58eb0c1f4939fe5ffa. 2024-12-02T09:21:41,127 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733131276213.e2c45d5b11370f58eb0c1f4939fe5ffa. after waiting 0 ms 2024-12-02T09:21:41,127 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733131276213.e2c45d5b11370f58eb0c1f4939fe5ffa. 2024-12-02T09:21:41,128 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733131276213.e2c45d5b11370f58eb0c1f4939fe5ffa.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/e2c45d5b11370f58eb0c1f4939fe5ffa/info/bb27cad513c94baca1abc85930a76f31.adb545fc7773af344dd36d23ae47da66->hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/adb545fc7773af344dd36d23ae47da66/info/bb27cad513c94baca1abc85930a76f31-bottom] to archive 2024-12-02T09:21:41,129 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733131276213.e2c45d5b11370f58eb0c1f4939fe5ffa.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-02T09:21:41,130 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733131276213.e2c45d5b11370f58eb0c1f4939fe5ffa.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/e2c45d5b11370f58eb0c1f4939fe5ffa/info/bb27cad513c94baca1abc85930a76f31.adb545fc7773af344dd36d23ae47da66 to hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/archive/data/default/TestLogRolling-testLogRolling/e2c45d5b11370f58eb0c1f4939fe5ffa/info/bb27cad513c94baca1abc85930a76f31.adb545fc7773af344dd36d23ae47da66 2024-12-02T09:21:41,130 WARN [StoreCloser-TestLogRolling-testLogRolling,,1733131276213.e2c45d5b11370f58eb0c1f4939fe5ffa.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [] 2024-12-02T09:21:41,134 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/data/default/TestLogRolling-testLogRolling/e2c45d5b11370f58eb0c1f4939fe5ffa/recovered.edits/90.seqid, newMaxSeqId=90, maxSeqId=85 2024-12-02T09:21:41,135 INFO [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733131276213.e2c45d5b11370f58eb0c1f4939fe5ffa. 2024-12-02T09:21:41,135 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e2c45d5b11370f58eb0c1f4939fe5ffa: Waiting for close lock at 1733131301127Running coprocessor pre-close hooks at 1733131301127Disabling compacts and flushes for region at 1733131301127Disabling writes for close at 1733131301127Writing region close event to WAL at 1733131301131 (+4 ms)Running coprocessor post-close hooks at 1733131301135 (+4 ms)Closed at 1733131301135 2024-12-02T09:21:41,135 DEBUG [RS_CLOSE_REGION-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1733131276213.e2c45d5b11370f58eb0c1f4939fe5ffa. 2024-12-02T09:21:41,161 INFO [regionserver/7c6d666a4939:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T09:21:41,196 INFO [regionserver/7c6d666a4939:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-02T09:21:41,196 INFO [regionserver/7c6d666a4939:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-02T09:21:41,294 INFO [RS:0;7c6d666a4939:37787 {}] regionserver.HRegionServer(976): stopping server 7c6d666a4939,37787,1733131252687; all regions closed. 2024-12-02T09:21:41,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:41,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:41,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:41,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:41,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:41,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741834_1010 (size=8107) 2024-12-02T09:21:41,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741834_1010 (size=8107) 2024-12-02T09:21:41,298 DEBUG [RS:0;7c6d666a4939:37787 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/oldWALs 2024-12-02T09:21:41,298 INFO [RS:0;7c6d666a4939:37787 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7c6d666a4939%2C37787%2C1733131252687.meta:.meta(num 1733131253679) 2024-12-02T09:21:41,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:41,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:41,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:41,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:41,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:41,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741881_1057 (size=778) 2024-12-02T09:21:41,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741881_1057 (size=778) 2024-12-02T09:21:41,480 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:41,482 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:41,703 DEBUG [RS:0;7c6d666a4939:37787 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/oldWALs 2024-12-02T09:21:41,703 INFO [RS:0;7c6d666a4939:37787 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7c6d666a4939%2C37787%2C1733131252687:(num 1733131301048) 2024-12-02T09:21:41,703 DEBUG [RS:0;7c6d666a4939:37787 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:21:41,703 INFO [RS:0;7c6d666a4939:37787 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T09:21:41,703 INFO [RS:0;7c6d666a4939:37787 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T09:21:41,704 INFO [RS:0;7c6d666a4939:37787 {}] hbase.ChoreService(370): Chore service for: regionserver/7c6d666a4939:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-02T09:21:41,704 INFO [RS:0;7c6d666a4939:37787 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T09:21:41,704 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T09:21:41,704 INFO [RS:0;7c6d666a4939:37787 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37787 2024-12-02T09:21:41,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T09:21:41,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1009a4a10d80001, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7c6d666a4939,37787,1733131252687 2024-12-02T09:21:41,740 INFO [RS:0;7c6d666a4939:37787 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T09:21:41,750 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7c6d666a4939,37787,1733131252687] 2024-12-02T09:21:41,758 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7c6d666a4939,37787,1733131252687 already deleted, retry=false 2024-12-02T09:21:41,758 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7c6d666a4939,37787,1733131252687 expired; onlineServers=0 2024-12-02T09:21:41,758 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7c6d666a4939,39695,1733131252558' ***** 2024-12-02T09:21:41,758 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-02T09:21:41,758 INFO [M:0;7c6d666a4939:39695 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T09:21:41,759 INFO [M:0;7c6d666a4939:39695 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T09:21:41,759 DEBUG [M:0;7c6d666a4939:39695 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-02T09:21:41,759 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-02T09:21:41,759 DEBUG [M:0;7c6d666a4939:39695 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-02T09:21:41,759 DEBUG [master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.large.0-1733131253040 {}] cleaner.HFileCleaner(306): Exit Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.large.0-1733131253040,5,FailOnTimeoutGroup] 2024-12-02T09:21:41,759 DEBUG [master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.small.0-1733131253041 {}] cleaner.HFileCleaner(306): Exit Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.small.0-1733131253041,5,FailOnTimeoutGroup] 2024-12-02T09:21:41,759 INFO [M:0;7c6d666a4939:39695 {}] hbase.ChoreService(370): Chore service for: master/7c6d666a4939:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-02T09:21:41,759 INFO [M:0;7c6d666a4939:39695 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T09:21:41,759 DEBUG [M:0;7c6d666a4939:39695 {}] master.HMaster(1795): Stopping service threads 2024-12-02T09:21:41,759 INFO [M:0;7c6d666a4939:39695 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-02T09:21:41,759 INFO [M:0;7c6d666a4939:39695 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T09:21:41,759 ERROR [M:0;7c6d666a4939:39695 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:32923,5,PEWorkerGroup] 2024-12-02T09:21:41,760 INFO [M:0;7c6d666a4939:39695 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T09:21:41,760 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-02T09:21:41,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-02T09:21:41,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:21:41,767 DEBUG [M:0;7c6d666a4939:39695 {}] zookeeper.ZKUtil(347): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-02T09:21:41,767 WARN [M:0;7c6d666a4939:39695 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-02T09:21:41,767 INFO [M:0;7c6d666a4939:39695 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/.lastflushedseqids 2024-12-02T09:21:41,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741882_1058 (size=228) 2024-12-02T09:21:41,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741882_1058 (size=228) 2024-12-02T09:21:41,772 INFO [M:0;7c6d666a4939:39695 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T09:21:41,773 INFO [M:0;7c6d666a4939:39695 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-02T09:21:41,773 DEBUG [M:0;7c6d666a4939:39695 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T09:21:41,773 INFO [M:0;7c6d666a4939:39695 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:21:41,773 DEBUG [M:0;7c6d666a4939:39695 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:21:41,773 DEBUG [M:0;7c6d666a4939:39695 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T09:21:41,773 DEBUG [M:0;7c6d666a4939:39695 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:21:41,773 INFO [M:0;7c6d666a4939:39695 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.43 KB heapSize=63.36 KB 2024-12-02T09:21:41,791 DEBUG [M:0;7c6d666a4939:39695 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a28e91c0c58a4d2cae0b6bc96e076486 is 82, key is hbase:meta,,1/info:regioninfo/1733131253700/Put/seqid=0 2024-12-02T09:21:41,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741883_1059 (size=5672) 2024-12-02T09:21:41,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741883_1059 (size=5672) 2024-12-02T09:21:41,796 INFO [M:0;7c6d666a4939:39695 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a28e91c0c58a4d2cae0b6bc96e076486 2024-12-02T09:21:41,815 DEBUG [M:0;7c6d666a4939:39695 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c994e3e706ca4d889bb9c9cf85b2e4a4 is 751, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733131254191/Put/seqid=0 2024-12-02T09:21:41,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741884_1060 (size=7091) 2024-12-02T09:21:41,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741884_1060 (size=7091) 2024-12-02T09:21:41,820 INFO [M:0;7c6d666a4939:39695 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.83 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c994e3e706ca4d889bb9c9cf85b2e4a4 2024-12-02T09:21:41,824 INFO [M:0;7c6d666a4939:39695 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c994e3e706ca4d889bb9c9cf85b2e4a4 2024-12-02T09:21:41,842 DEBUG [M:0;7c6d666a4939:39695 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/caa87adb1a21496a9c59ce6fcc51ab81 is 69, key is 7c6d666a4939,37787,1733131252687/rs:state/1733131253140/Put/seqid=0 2024-12-02T09:21:41,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741885_1061 (size=5156) 2024-12-02T09:21:41,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741885_1061 (size=5156) 2024-12-02T09:21:41,848 INFO [M:0;7c6d666a4939:39695 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/caa87adb1a21496a9c59ce6fcc51ab81 2024-12-02T09:21:41,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1009a4a10d80001, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:21:41,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1009a4a10d80001, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:21:41,850 INFO [RS:0;7c6d666a4939:37787 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T09:21:41,850 INFO [RS:0;7c6d666a4939:37787 {}] regionserver.HRegionServer(1031): Exiting; stopping=7c6d666a4939,37787,1733131252687; zookeeper connection closed. 2024-12-02T09:21:41,851 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@70ed0a34 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@70ed0a34 2024-12-02T09:21:41,851 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-02T09:21:41,871 DEBUG [M:0;7c6d666a4939:39695 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1adb5a69ca6547d2a075b62f396dd276 is 52, key is load_balancer_on/state:d/1733131253819/Put/seqid=0 2024-12-02T09:21:41,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741886_1062 (size=5056) 2024-12-02T09:21:41,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741886_1062 (size=5056) 2024-12-02T09:21:41,875 INFO [M:0;7c6d666a4939:39695 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1adb5a69ca6547d2a075b62f396dd276 2024-12-02T09:21:41,880 DEBUG [M:0;7c6d666a4939:39695 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a28e91c0c58a4d2cae0b6bc96e076486 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a28e91c0c58a4d2cae0b6bc96e076486 2024-12-02T09:21:41,885 INFO [M:0;7c6d666a4939:39695 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a28e91c0c58a4d2cae0b6bc96e076486, entries=8, sequenceid=125, filesize=5.5 K 2024-12-02T09:21:41,886 DEBUG [M:0;7c6d666a4939:39695 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c994e3e706ca4d889bb9c9cf85b2e4a4 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c994e3e706ca4d889bb9c9cf85b2e4a4 2024-12-02T09:21:41,891 INFO [M:0;7c6d666a4939:39695 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c994e3e706ca4d889bb9c9cf85b2e4a4 2024-12-02T09:21:41,891 INFO [M:0;7c6d666a4939:39695 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c994e3e706ca4d889bb9c9cf85b2e4a4, entries=13, sequenceid=125, filesize=6.9 K 2024-12-02T09:21:41,892 DEBUG [M:0;7c6d666a4939:39695 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/caa87adb1a21496a9c59ce6fcc51ab81 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/caa87adb1a21496a9c59ce6fcc51ab81 2024-12-02T09:21:41,895 INFO [M:0;7c6d666a4939:39695 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/caa87adb1a21496a9c59ce6fcc51ab81, entries=1, sequenceid=125, filesize=5.0 K 2024-12-02T09:21:41,896 DEBUG [M:0;7c6d666a4939:39695 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1adb5a69ca6547d2a075b62f396dd276 as hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/1adb5a69ca6547d2a075b62f396dd276 2024-12-02T09:21:41,900 INFO [M:0;7c6d666a4939:39695 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:32923/user/jenkins/test-data/00f2d198-6add-ea68-60be-8398cbfbde38/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/1adb5a69ca6547d2a075b62f396dd276, entries=1, sequenceid=125, filesize=4.9 K 2024-12-02T09:21:41,901 INFO [M:0;7c6d666a4939:39695 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.43 KB/52663, heapSize ~63.30 KB/64816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 128ms, sequenceid=125, compaction requested=false 2024-12-02T09:21:41,902 INFO [M:0;7c6d666a4939:39695 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:21:41,902 DEBUG [M:0;7c6d666a4939:39695 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733131301773Disabling compacts and flushes for region at 1733131301773Disabling writes for close at 1733131301773Obtaining lock to block concurrent updates at 1733131301773Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733131301773Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52663, getHeapSize=64816, getOffHeapSize=0, getCellsCount=148 at 1733131301774 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733131301774Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733131301774Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733131301791 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733131301791Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733131301800 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733131301814 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733131301815 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733131301825 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733131301841 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733131301842 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733131301852 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733131301870 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733131301870Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e669b9e: reopening flushed file at 1733131301879 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66dbd7e8: reopening flushed file at 1733131301885 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@245b620c: reopening flushed file at 1733131301891 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@462522e6: reopening flushed file at 1733131301895 (+4 ms)Finished flush of dataSize ~51.43 KB/52663, heapSize ~63.30 KB/64816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 128ms, sequenceid=125, compaction requested=false at 1733131301901 (+6 ms)Writing region close event to WAL at 1733131301902 (+1 ms)Closed at 1733131301902 2024-12-02T09:21:41,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:41,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:41,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:41,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:41,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:41,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42295 is added to blk_1073741830_1006 (size=61332) 2024-12-02T09:21:41,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42255 is added to blk_1073741830_1006 (size=61332) 2024-12-02T09:21:41,905 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T09:21:41,905 INFO [M:0;7c6d666a4939:39695 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-02T09:21:41,905 INFO [M:0;7c6d666a4939:39695 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39695 2024-12-02T09:21:41,906 INFO [M:0;7c6d666a4939:39695 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T09:21:42,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:21:42,014 INFO [M:0;7c6d666a4939:39695 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T09:21:42,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39695-0x1009a4a10d80000, quorum=127.0.0.1:62506, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:21:42,017 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1d3b5490{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:21:42,018 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@41b3b520{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:21:42,018 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:21:42,018 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@369abda8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:21:42,018 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@31f94774{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/hadoop.log.dir/,STOPPED} 2024-12-02T09:21:42,020 WARN [BP-1798352675-172.17.0.3-1733131250135 heartbeating to localhost/127.0.0.1:32923 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T09:21:42,020 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T09:21:42,020 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T09:21:42,020 WARN [BP-1798352675-172.17.0.3-1733131250135 heartbeating to localhost/127.0.0.1:32923 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1798352675-172.17.0.3-1733131250135 (Datanode Uuid bdffc36d-9e37-406e-9f09-f3b04c97f328) service to localhost/127.0.0.1:32923 2024-12-02T09:21:42,021 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/cluster_ba17a36c-de3c-0319-27d7-59df5dbdde8b/data/data3/current/BP-1798352675-172.17.0.3-1733131250135 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:21:42,021 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/cluster_ba17a36c-de3c-0319-27d7-59df5dbdde8b/data/data4/current/BP-1798352675-172.17.0.3-1733131250135 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:21:42,021 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T09:21:42,023 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5eab25ce{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:21:42,024 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5773e0ea{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:21:42,024 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:21:42,024 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70aed17c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:21:42,024 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@208716aa{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/hadoop.log.dir/,STOPPED} 2024-12-02T09:21:42,025 WARN [BP-1798352675-172.17.0.3-1733131250135 heartbeating to localhost/127.0.0.1:32923 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T09:21:42,025 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T09:21:42,025 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T09:21:42,025 WARN [BP-1798352675-172.17.0.3-1733131250135 heartbeating to localhost/127.0.0.1:32923 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1798352675-172.17.0.3-1733131250135 (Datanode Uuid b905a314-56b7-453b-afa4-4a480bcd3fea) service to localhost/127.0.0.1:32923 2024-12-02T09:21:42,026 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/cluster_ba17a36c-de3c-0319-27d7-59df5dbdde8b/data/data1/current/BP-1798352675-172.17.0.3-1733131250135 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:21:42,026 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/cluster_ba17a36c-de3c-0319-27d7-59df5dbdde8b/data/data2/current/BP-1798352675-172.17.0.3-1733131250135 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:21:42,027 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T09:21:42,033 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@35a5806e{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T09:21:42,034 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@191b8d86{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:21:42,034 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:21:42,034 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1d428ad5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:21:42,034 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@811037{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/hadoop.log.dir/,STOPPED} 2024-12-02T09:21:42,044 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-02T09:21:42,076 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-02T09:21:42,089 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=230 (was 204) Potentially hanging thread: nioEventLoopGroup-40-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:32923 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:32923 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-39-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:32923 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: HMaster-EventLoopGroup-14-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-14-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-41-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:32923 from jenkins.hfs.6 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:32923 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-40-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-38-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-39-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:32923 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-38-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:32923 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-40-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-38-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-39-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-14-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-41-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:32923 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-41-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:32923 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=518 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=227 (was 222) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=451 (was 1170) 2024-12-02T09:21:42,097 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=230, OpenFileDescriptor=518, MaxFileDescriptor=1048576, SystemLoadAverage=227, ProcessCount=11, AvailableMemoryMB=451 2024-12-02T09:21:42,097 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-02T09:21:42,097 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/hadoop.log.dir so I do NOT create it in target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f 2024-12-02T09:21:42,097 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffe5a635-23cf-0358-fbb6-163d663cdcce/hadoop.tmp.dir so I do NOT create it in target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f 2024-12-02T09:21:42,097 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/cluster_62bbdbaa-2ad6-42c0-7b1c-03d67d162a57, deleteOnExit=true 2024-12-02T09:21:42,097 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-02T09:21:42,097 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/test.cache.data in system properties and HBase conf 2024-12-02T09:21:42,098 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/hadoop.tmp.dir in system properties and HBase conf 2024-12-02T09:21:42,098 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/hadoop.log.dir in system properties and HBase conf 2024-12-02T09:21:42,098 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-02T09:21:42,098 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-02T09:21:42,098 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-02T09:21:42,098 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-02T09:21:42,098 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-02T09:21:42,098 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-02T09:21:42,098 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-02T09:21:42,098 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T09:21:42,098 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-02T09:21:42,099 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-02T09:21:42,099 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T09:21:42,099 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T09:21:42,099 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-02T09:21:42,099 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/nfs.dump.dir in system properties and HBase conf 2024-12-02T09:21:42,099 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/java.io.tmpdir in system properties and HBase conf 2024-12-02T09:21:42,099 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T09:21:42,099 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-02T09:21:42,099 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-02T09:21:42,114 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-02T09:21:42,307 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:21:42,310 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:21:42,311 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:21:42,311 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:21:42,311 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T09:21:42,312 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:21:42,312 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@19f4d4a5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:21:42,312 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@512a083f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:21:42,416 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7ffc1e9{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/java.io.tmpdir/jetty-localhost-34985-hadoop-hdfs-3_4_1-tests_jar-_-any-17518719933917394194/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T09:21:42,417 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5e2fef96{HTTP/1.1, (http/1.1)}{localhost:34985} 2024-12-02T09:21:42,417 INFO [Time-limited test {}] server.Server(415): Started @298898ms 2024-12-02T09:21:42,430 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-12-02T09:21:42,481 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:42,482 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:42,592 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:21:42,594 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:21:42,595 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:21:42,595 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:21:42,595 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T09:21:42,595 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@624ed4c3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:21:42,595 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7acee9f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:21:42,688 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1efd209{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/java.io.tmpdir/jetty-localhost-46097-hadoop-hdfs-3_4_1-tests_jar-_-any-11224120883906832139/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:21:42,688 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@49e6dd92{HTTP/1.1, (http/1.1)}{localhost:46097} 2024-12-02T09:21:42,688 INFO [Time-limited test {}] server.Server(415): Started @299169ms 2024-12-02T09:21:42,689 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T09:21:42,711 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T09:21:42,714 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T09:21:42,715 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T09:21:42,715 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T09:21:42,715 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T09:21:42,715 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1cfa2b02{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/hadoop.log.dir/,AVAILABLE} 2024-12-02T09:21:42,716 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@f9db876{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T09:21:42,813 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@335e1da0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/java.io.tmpdir/jetty-localhost-45793-hadoop-hdfs-3_4_1-tests_jar-_-any-17736553566702248003/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:21:42,813 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@9279e7{HTTP/1.1, (http/1.1)}{localhost:45793} 2024-12-02T09:21:42,813 INFO [Time-limited test {}] server.Server(415): Started @299294ms 2024-12-02T09:21:42,814 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T09:21:43,481 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:43,483 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:43,577 WARN [Thread-2491 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/cluster_62bbdbaa-2ad6-42c0-7b1c-03d67d162a57/data/data1/current/BP-1838583546-172.17.0.3-1733131302118/current, will proceed with Du for space computation calculation, 2024-12-02T09:21:43,577 WARN [Thread-2492 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/cluster_62bbdbaa-2ad6-42c0-7b1c-03d67d162a57/data/data2/current/BP-1838583546-172.17.0.3-1733131302118/current, will proceed with Du for space computation calculation, 2024-12-02T09:21:43,595 WARN [Thread-2455 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T09:21:43,596 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x156435164d8ffec9 with lease ID 0x2ab5eb5f1fb6cd38: Processing first storage report for DS-0541e858-d72d-4cf8-a8c4-fcc96ca9fd38 from datanode DatanodeRegistration(127.0.0.1:36095, datanodeUuid=6a262b8d-8647-4eaa-a35d-a3d243983b6b, infoPort=46443, infoSecurePort=0, ipcPort=44049, storageInfo=lv=-57;cid=testClusterID;nsid=477212123;c=1733131302118) 2024-12-02T09:21:43,597 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x156435164d8ffec9 with lease ID 0x2ab5eb5f1fb6cd38: from storage DS-0541e858-d72d-4cf8-a8c4-fcc96ca9fd38 node DatanodeRegistration(127.0.0.1:36095, datanodeUuid=6a262b8d-8647-4eaa-a35d-a3d243983b6b, infoPort=46443, infoSecurePort=0, ipcPort=44049, storageInfo=lv=-57;cid=testClusterID;nsid=477212123;c=1733131302118), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:21:43,597 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x156435164d8ffec9 with lease ID 0x2ab5eb5f1fb6cd38: Processing first storage report for DS-186f5cf1-d699-4479-b38a-a9c8d3654cf8 from datanode DatanodeRegistration(127.0.0.1:36095, datanodeUuid=6a262b8d-8647-4eaa-a35d-a3d243983b6b, infoPort=46443, infoSecurePort=0, ipcPort=44049, storageInfo=lv=-57;cid=testClusterID;nsid=477212123;c=1733131302118) 2024-12-02T09:21:43,597 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x156435164d8ffec9 with lease ID 0x2ab5eb5f1fb6cd38: from storage DS-186f5cf1-d699-4479-b38a-a9c8d3654cf8 node DatanodeRegistration(127.0.0.1:36095, datanodeUuid=6a262b8d-8647-4eaa-a35d-a3d243983b6b, infoPort=46443, infoSecurePort=0, ipcPort=44049, storageInfo=lv=-57;cid=testClusterID;nsid=477212123;c=1733131302118), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:21:43,726 WARN [Thread-2503 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/cluster_62bbdbaa-2ad6-42c0-7b1c-03d67d162a57/data/data4/current/BP-1838583546-172.17.0.3-1733131302118/current, will proceed with Du for space computation calculation, 2024-12-02T09:21:43,726 WARN [Thread-2502 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/cluster_62bbdbaa-2ad6-42c0-7b1c-03d67d162a57/data/data3/current/BP-1838583546-172.17.0.3-1733131302118/current, will proceed with Du for space computation calculation, 2024-12-02T09:21:43,751 WARN [Thread-2478 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T09:21:43,759 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x793b19fa98d9c54f with lease ID 0x2ab5eb5f1fb6cd39: Processing first storage report for DS-d34ed648-5ca9-490f-9619-28d85e19d967 from datanode DatanodeRegistration(127.0.0.1:38373, datanodeUuid=e66fb798-a919-4627-bb84-7b637111a807, infoPort=45285, infoSecurePort=0, ipcPort=36929, storageInfo=lv=-57;cid=testClusterID;nsid=477212123;c=1733131302118) 2024-12-02T09:21:43,759 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x793b19fa98d9c54f with lease ID 0x2ab5eb5f1fb6cd39: from storage DS-d34ed648-5ca9-490f-9619-28d85e19d967 node DatanodeRegistration(127.0.0.1:38373, datanodeUuid=e66fb798-a919-4627-bb84-7b637111a807, infoPort=45285, infoSecurePort=0, ipcPort=36929, storageInfo=lv=-57;cid=testClusterID;nsid=477212123;c=1733131302118), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:21:43,759 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x793b19fa98d9c54f with lease ID 0x2ab5eb5f1fb6cd39: Processing first storage report for DS-b39d1173-d673-48d8-97b6-e52cf786b001 from datanode DatanodeRegistration(127.0.0.1:38373, datanodeUuid=e66fb798-a919-4627-bb84-7b637111a807, infoPort=45285, infoSecurePort=0, ipcPort=36929, storageInfo=lv=-57;cid=testClusterID;nsid=477212123;c=1733131302118) 2024-12-02T09:21:43,760 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x793b19fa98d9c54f with lease ID 0x2ab5eb5f1fb6cd39: from storage DS-b39d1173-d673-48d8-97b6-e52cf786b001 node DatanodeRegistration(127.0.0.1:38373, datanodeUuid=e66fb798-a919-4627-bb84-7b637111a807, infoPort=45285, infoSecurePort=0, ipcPort=36929, storageInfo=lv=-57;cid=testClusterID;nsid=477212123;c=1733131302118), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T09:21:43,839 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f 2024-12-02T09:21:43,842 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/cluster_62bbdbaa-2ad6-42c0-7b1c-03d67d162a57/zookeeper_0, clientPort=59473, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/cluster_62bbdbaa-2ad6-42c0-7b1c-03d67d162a57/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/cluster_62bbdbaa-2ad6-42c0-7b1c-03d67d162a57/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-02T09:21:43,843 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59473 2024-12-02T09:21:43,843 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:21:43,845 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:21:43,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38373 is added to blk_1073741825_1001 (size=7) 2024-12-02T09:21:43,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36095 is added to blk_1073741825_1001 (size=7) 2024-12-02T09:21:43,871 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8 with version=8 2024-12-02T09:21:43,871 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:37857/user/jenkins/test-data/7e1737a3-7e66-7fcb-bc19-3c4d5d3e419a/hbase-staging 2024-12-02T09:21:43,874 INFO [Time-limited test {}] client.ConnectionUtils(128): master/7c6d666a4939:0 server-side Connection retries=45 2024-12-02T09:21:43,874 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:21:43,874 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T09:21:43,874 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T09:21:43,874 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:21:43,874 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T09:21:43,874 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-02T09:21:43,874 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T09:21:43,876 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35331 2024-12-02T09:21:43,877 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35331 connecting to ZooKeeper ensemble=127.0.0.1:59473 2024-12-02T09:21:43,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:353310x0, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T09:21:43,944 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35331-0x1009a4adad50000 connected 2024-12-02T09:21:44,022 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:21:44,024 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:21:44,026 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:21:44,027 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8, hbase.cluster.distributed=false 2024-12-02T09:21:44,029 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T09:21:44,029 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35331 2024-12-02T09:21:44,030 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35331 2024-12-02T09:21:44,030 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35331 2024-12-02T09:21:44,030 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35331 2024-12-02T09:21:44,030 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35331 2024-12-02T09:21:44,050 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/7c6d666a4939:0 server-side Connection retries=45 2024-12-02T09:21:44,050 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:21:44,050 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T09:21:44,050 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T09:21:44,050 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T09:21:44,050 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T09:21:44,050 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T09:21:44,050 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T09:21:44,051 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40561 2024-12-02T09:21:44,053 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40561 connecting to ZooKeeper ensemble=127.0.0.1:59473 2024-12-02T09:21:44,054 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:21:44,056 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:21:44,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:405610x0, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T09:21:44,069 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40561-0x1009a4adad50001 connected 2024-12-02T09:21:44,069 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40561-0x1009a4adad50001, quorum=127.0.0.1:59473, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:21:44,070 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T09:21:44,070 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T09:21:44,071 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40561-0x1009a4adad50001, quorum=127.0.0.1:59473, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T09:21:44,071 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40561-0x1009a4adad50001, quorum=127.0.0.1:59473, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T09:21:44,072 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40561 2024-12-02T09:21:44,073 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40561 2024-12-02T09:21:44,075 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40561 2024-12-02T09:21:44,075 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40561 2024-12-02T09:21:44,075 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40561 2024-12-02T09:21:44,090 DEBUG [M:0;7c6d666a4939:35331 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;7c6d666a4939:35331 2024-12-02T09:21:44,091 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/7c6d666a4939,35331,1733131303873 2024-12-02T09:21:44,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x1009a4adad50001, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:21:44,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:21:44,102 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/7c6d666a4939,35331,1733131303873 2024-12-02T09:21:44,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:21:44,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x1009a4adad50001, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T09:21:44,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x1009a4adad50001, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:21:44,111 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-02T09:21:44,111 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/7c6d666a4939,35331,1733131303873 from backup master directory 2024-12-02T09:21:44,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x1009a4adad50001, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:21:44,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/7c6d666a4939,35331,1733131303873 2024-12-02T09:21:44,119 WARN [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T09:21:44,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T09:21:44,119 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=7c6d666a4939,35331,1733131303873 2024-12-02T09:21:44,131 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/hbase.id] with ID: 9c7b5800-9400-4147-b096-99af472fc84e 2024-12-02T09:21:44,131 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/.tmp/hbase.id 2024-12-02T09:21:44,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36095 is added to blk_1073741826_1002 (size=42) 2024-12-02T09:21:44,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38373 is added to blk_1073741826_1002 (size=42) 2024-12-02T09:21:44,139 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/.tmp/hbase.id]:[hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/hbase.id] 2024-12-02T09:21:44,164 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:21:44,165 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-02T09:21:44,167 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-12-02T09:21:44,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x1009a4adad50001, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:21:44,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:21:44,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38373 is added to blk_1073741827_1003 (size=196) 2024-12-02T09:21:44,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36095 is added to blk_1073741827_1003 (size=196) 2024-12-02T09:21:44,205 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T09:21:44,206 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-02T09:21:44,211 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-02T09:21:44,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38373 is added to blk_1073741828_1004 (size=1189) 2024-12-02T09:21:44,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36095 is added to blk_1073741828_1004 (size=1189) 2024-12-02T09:21:44,218 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/data/master/store 2024-12-02T09:21:44,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36095 is added to blk_1073741829_1005 (size=34) 2024-12-02T09:21:44,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38373 is added to blk_1073741829_1005 (size=34) 2024-12-02T09:21:44,229 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:21:44,229 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T09:21:44,229 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:21:44,229 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:21:44,229 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T09:21:44,229 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:21:44,229 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:21:44,229 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733131304229Disabling compacts and flushes for region at 1733131304229Disabling writes for close at 1733131304229Writing region close event to WAL at 1733131304229Closed at 1733131304229 2024-12-02T09:21:44,230 WARN [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/data/master/store/.initializing 2024-12-02T09:21:44,230 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/WALs/7c6d666a4939,35331,1733131303873 2024-12-02T09:21:44,233 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c6d666a4939%2C35331%2C1733131303873, suffix=, logDir=hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/WALs/7c6d666a4939,35331,1733131303873, archiveDir=hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/oldWALs, maxLogs=10 2024-12-02T09:21:44,233 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C35331%2C1733131303873.1733131304233 2024-12-02T09:21:44,248 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/WALs/7c6d666a4939,35331,1733131303873/7c6d666a4939%2C35331%2C1733131303873.1733131304233 2024-12-02T09:21:44,252 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46443:46443),(127.0.0.1/127.0.0.1:45285:45285)] 2024-12-02T09:21:44,259 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-02T09:21:44,259 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:21:44,260 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:21:44,260 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:21:44,263 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:21:44,265 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-02T09:21:44,265 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:21:44,265 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:21:44,266 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:21:44,267 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-02T09:21:44,267 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:21:44,267 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:21:44,267 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:21:44,269 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-02T09:21:44,269 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:21:44,269 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:21:44,269 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:21:44,270 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-02T09:21:44,270 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:21:44,271 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T09:21:44,271 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:21:44,272 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:21:44,272 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:21:44,273 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:21:44,273 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:21:44,273 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T09:21:44,275 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T09:21:44,283 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T09:21:44,284 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=823526, jitterRate=0.04716747999191284}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T09:21:44,285 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733131304260Initializing all the Stores at 1733131304261 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131304261Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131304263 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131304263Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131304263Cleaning up temporary data from old regions at 1733131304273 (+10 ms)Region opened successfully at 1733131304285 (+12 ms) 2024-12-02T09:21:44,287 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-02T09:21:44,293 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1155c92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7c6d666a4939/172.17.0.3:0 2024-12-02T09:21:44,295 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-02T09:21:44,295 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-02T09:21:44,295 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-02T09:21:44,295 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T09:21:44,296 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-02T09:21:44,296 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-02T09:21:44,296 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-02T09:21:44,309 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-02T09:21:44,310 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-02T09:21:44,318 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-02T09:21:44,319 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-02T09:21:44,321 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-02T09:21:44,330 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-02T09:21:44,331 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-02T09:21:44,333 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-02T09:21:44,343 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-02T09:21:44,344 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-02T09:21:44,352 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-02T09:21:44,354 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-02T09:21:44,364 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-02T09:21:44,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T09:21:44,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:21:44,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x1009a4adad50001, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T09:21:44,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x1009a4adad50001, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:21:44,373 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=7c6d666a4939,35331,1733131303873, sessionid=0x1009a4adad50000, setting cluster-up flag (Was=false) 2024-12-02T09:21:44,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:21:44,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x1009a4adad50001, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:21:44,414 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-02T09:21:44,415 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7c6d666a4939,35331,1733131303873 2024-12-02T09:21:44,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x1009a4adad50001, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:21:44,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:21:44,472 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-02T09:21:44,474 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=7c6d666a4939,35331,1733131303873 2024-12-02T09:21:44,476 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-02T09:21:44,477 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T09:21:44,478 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-02T09:21:44,478 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-02T09:21:44,478 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 7c6d666a4939,35331,1733131303873 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-02T09:21:44,480 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:21:44,480 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:21:44,480 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:21:44,480 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/7c6d666a4939:0, corePoolSize=5, maxPoolSize=5 2024-12-02T09:21:44,480 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/7c6d666a4939:0, corePoolSize=10, maxPoolSize=10 2024-12-02T09:21:44,480 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:21:44,480 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/7c6d666a4939:0, corePoolSize=2, maxPoolSize=2 2024-12-02T09:21:44,480 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:21:44,482 INFO [RS:0;7c6d666a4939:40561 {}] regionserver.HRegionServer(746): ClusterId : 9c7b5800-9400-4147-b096-99af472fc84e 2024-12-02T09:21:44,482 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:44,482 DEBUG [RS:0;7c6d666a4939:40561 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T09:21:44,483 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T09:21:44,483 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-02T09:21:44,483 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:44,487 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:21:44,487 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-02T09:21:44,489 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733131334489 2024-12-02T09:21:44,489 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-02T09:21:44,490 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-02T09:21:44,490 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-02T09:21:44,490 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-02T09:21:44,490 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-02T09:21:44,490 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-02T09:21:44,490 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T09:21:44,491 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-02T09:21:44,491 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-02T09:21:44,491 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-02T09:21:44,491 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-02T09:21:44,491 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-02T09:21:44,492 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.large.0-1733131304491,5,FailOnTimeoutGroup] 2024-12-02T09:21:44,493 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.small.0-1733131304492,5,FailOnTimeoutGroup] 2024-12-02T09:21:44,493 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T09:21:44,493 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-02T09:21:44,493 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-02T09:21:44,493 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-02T09:21:44,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38373 is added to blk_1073741831_1007 (size=1321) 2024-12-02T09:21:44,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36095 is added to blk_1073741831_1007 (size=1321) 2024-12-02T09:21:44,495 DEBUG [RS:0;7c6d666a4939:40561 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T09:21:44,495 DEBUG [RS:0;7c6d666a4939:40561 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T09:21:44,495 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-02T09:21:44,495 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8 2024-12-02T09:21:44,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38373 is added to blk_1073741832_1008 (size=32) 2024-12-02T09:21:44,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36095 is added to blk_1073741832_1008 (size=32) 2024-12-02T09:21:44,502 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:21:44,503 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T09:21:44,504 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T09:21:44,504 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:21:44,505 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:21:44,505 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T09:21:44,506 DEBUG [RS:0;7c6d666a4939:40561 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T09:21:44,506 DEBUG [RS:0;7c6d666a4939:40561 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32373b3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=7c6d666a4939/172.17.0.3:0 2024-12-02T09:21:44,507 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T09:21:44,507 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:21:44,507 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:21:44,507 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T09:21:44,509 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T09:21:44,509 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:21:44,509 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:21:44,509 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T09:21:44,511 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T09:21:44,511 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:21:44,511 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:21:44,511 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T09:21:44,512 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/data/hbase/meta/1588230740 2024-12-02T09:21:44,512 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/data/hbase/meta/1588230740 2024-12-02T09:21:44,513 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T09:21:44,513 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T09:21:44,514 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T09:21:44,515 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T09:21:44,517 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T09:21:44,517 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=812317, jitterRate=0.03291487693786621}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T09:21:44,518 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733131304502Initializing all the Stores at 1733131304502Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131304502Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131304503 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131304503Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131304503Cleaning up temporary data from old regions at 1733131304513 (+10 ms)Region opened successfully at 1733131304518 (+5 ms) 2024-12-02T09:21:44,518 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T09:21:44,518 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T09:21:44,518 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T09:21:44,518 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T09:21:44,518 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T09:21:44,518 DEBUG [RS:0;7c6d666a4939:40561 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;7c6d666a4939:40561 2024-12-02T09:21:44,518 INFO [RS:0;7c6d666a4939:40561 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T09:21:44,518 INFO [RS:0;7c6d666a4939:40561 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T09:21:44,519 DEBUG [RS:0;7c6d666a4939:40561 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T09:21:44,519 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T09:21:44,519 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733131304518Disabling compacts and flushes for region at 1733131304518Disabling writes for close at 1733131304518Writing region close event to WAL at 1733131304519 (+1 ms)Closed at 1733131304519 2024-12-02T09:21:44,519 INFO [RS:0;7c6d666a4939:40561 {}] regionserver.HRegionServer(2659): reportForDuty to master=7c6d666a4939,35331,1733131303873 with port=40561, startcode=1733131304049 2024-12-02T09:21:44,519 DEBUG [RS:0;7c6d666a4939:40561 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T09:21:44,520 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T09:21:44,520 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-02T09:21:44,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-02T09:21:44,521 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T09:21:44,522 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-02T09:21:44,522 INFO [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50183, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T09:21:44,522 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35331 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 7c6d666a4939,40561,1733131304049 2024-12-02T09:21:44,523 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35331 {}] master.ServerManager(517): Registering regionserver=7c6d666a4939,40561,1733131304049 2024-12-02T09:21:44,524 DEBUG [RS:0;7c6d666a4939:40561 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8 2024-12-02T09:21:44,524 DEBUG [RS:0;7c6d666a4939:40561 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44613 2024-12-02T09:21:44,524 DEBUG [RS:0;7c6d666a4939:40561 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T09:21:44,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T09:21:44,531 DEBUG [RS:0;7c6d666a4939:40561 {}] zookeeper.ZKUtil(111): regionserver:40561-0x1009a4adad50001, quorum=127.0.0.1:59473, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/7c6d666a4939,40561,1733131304049 2024-12-02T09:21:44,531 WARN [RS:0;7c6d666a4939:40561 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T09:21:44,531 INFO [RS:0;7c6d666a4939:40561 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-02T09:21:44,531 DEBUG [RS:0;7c6d666a4939:40561 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/WALs/7c6d666a4939,40561,1733131304049 2024-12-02T09:21:44,531 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [7c6d666a4939,40561,1733131304049] 2024-12-02T09:21:44,534 INFO [RS:0;7c6d666a4939:40561 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T09:21:44,536 INFO [RS:0;7c6d666a4939:40561 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T09:21:44,536 INFO [RS:0;7c6d666a4939:40561 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T09:21:44,536 INFO [RS:0;7c6d666a4939:40561 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:21:44,536 INFO [RS:0;7c6d666a4939:40561 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T09:21:44,537 INFO [RS:0;7c6d666a4939:40561 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T09:21:44,537 INFO [RS:0;7c6d666a4939:40561 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T09:21:44,538 DEBUG [RS:0;7c6d666a4939:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:21:44,538 DEBUG [RS:0;7c6d666a4939:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:21:44,538 DEBUG [RS:0;7c6d666a4939:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:21:44,538 DEBUG [RS:0;7c6d666a4939:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:21:44,538 DEBUG [RS:0;7c6d666a4939:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:21:44,538 DEBUG [RS:0;7c6d666a4939:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/7c6d666a4939:0, corePoolSize=2, maxPoolSize=2 2024-12-02T09:21:44,538 DEBUG [RS:0;7c6d666a4939:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:21:44,538 DEBUG [RS:0;7c6d666a4939:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:21:44,538 DEBUG [RS:0;7c6d666a4939:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:21:44,538 DEBUG [RS:0;7c6d666a4939:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:21:44,538 DEBUG [RS:0;7c6d666a4939:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:21:44,538 DEBUG [RS:0;7c6d666a4939:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/7c6d666a4939:0, corePoolSize=1, maxPoolSize=1 2024-12-02T09:21:44,538 DEBUG [RS:0;7c6d666a4939:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/7c6d666a4939:0, corePoolSize=3, maxPoolSize=3 2024-12-02T09:21:44,538 DEBUG [RS:0;7c6d666a4939:40561 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/7c6d666a4939:0, corePoolSize=3, maxPoolSize=3 2024-12-02T09:21:44,538 INFO [RS:0;7c6d666a4939:40561 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T09:21:44,539 INFO [RS:0;7c6d666a4939:40561 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T09:21:44,539 INFO [RS:0;7c6d666a4939:40561 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:21:44,539 INFO [RS:0;7c6d666a4939:40561 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T09:21:44,539 INFO [RS:0;7c6d666a4939:40561 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T09:21:44,539 INFO [RS:0;7c6d666a4939:40561 {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,40561,1733131304049-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T09:21:44,552 INFO [RS:0;7c6d666a4939:40561 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T09:21:44,553 INFO [RS:0;7c6d666a4939:40561 {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,40561,1733131304049-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:21:44,553 INFO [RS:0;7c6d666a4939:40561 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:21:44,553 INFO [RS:0;7c6d666a4939:40561 {}] regionserver.Replication(171): 7c6d666a4939,40561,1733131304049 started 2024-12-02T09:21:44,568 INFO [RS:0;7c6d666a4939:40561 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:21:44,568 INFO [RS:0;7c6d666a4939:40561 {}] regionserver.HRegionServer(1482): Serving as 7c6d666a4939,40561,1733131304049, RpcServer on 7c6d666a4939/172.17.0.3:40561, sessionid=0x1009a4adad50001 2024-12-02T09:21:44,568 DEBUG [RS:0;7c6d666a4939:40561 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T09:21:44,568 DEBUG [RS:0;7c6d666a4939:40561 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 7c6d666a4939,40561,1733131304049 2024-12-02T09:21:44,568 DEBUG [RS:0;7c6d666a4939:40561 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7c6d666a4939,40561,1733131304049' 2024-12-02T09:21:44,568 DEBUG [RS:0;7c6d666a4939:40561 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T09:21:44,569 DEBUG [RS:0;7c6d666a4939:40561 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T09:21:44,569 DEBUG [RS:0;7c6d666a4939:40561 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T09:21:44,569 DEBUG [RS:0;7c6d666a4939:40561 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T09:21:44,569 DEBUG [RS:0;7c6d666a4939:40561 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 7c6d666a4939,40561,1733131304049 2024-12-02T09:21:44,569 DEBUG [RS:0;7c6d666a4939:40561 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '7c6d666a4939,40561,1733131304049' 2024-12-02T09:21:44,569 DEBUG [RS:0;7c6d666a4939:40561 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T09:21:44,570 DEBUG [RS:0;7c6d666a4939:40561 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T09:21:44,570 DEBUG [RS:0;7c6d666a4939:40561 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T09:21:44,570 INFO [RS:0;7c6d666a4939:40561 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T09:21:44,570 INFO [RS:0;7c6d666a4939:40561 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T09:21:44,672 WARN [7c6d666a4939:35331 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-02T09:21:44,673 INFO [RS:0;7c6d666a4939:40561 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c6d666a4939%2C40561%2C1733131304049, suffix=, logDir=hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/WALs/7c6d666a4939,40561,1733131304049, archiveDir=hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/oldWALs, maxLogs=32 2024-12-02T09:21:44,674 INFO [RS:0;7c6d666a4939:40561 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C40561%2C1733131304049.1733131304674 2024-12-02T09:21:44,684 INFO [RS:0;7c6d666a4939:40561 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/WALs/7c6d666a4939,40561,1733131304049/7c6d666a4939%2C40561%2C1733131304049.1733131304674 2024-12-02T09:21:44,685 DEBUG [RS:0;7c6d666a4939:40561 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45285:45285),(127.0.0.1/127.0.0.1:46443:46443)] 2024-12-02T09:21:44,923 DEBUG [7c6d666a4939:35331 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-02T09:21:44,923 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=7c6d666a4939,40561,1733131304049 2024-12-02T09:21:44,924 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7c6d666a4939,40561,1733131304049, state=OPENING 2024-12-02T09:21:44,964 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-02T09:21:44,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x1009a4adad50001, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:21:44,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:21:44,973 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:21:44,973 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:21:44,973 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T09:21:44,973 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=7c6d666a4939,40561,1733131304049}] 2024-12-02T09:21:45,126 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T09:21:45,133 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54483, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T09:21:45,147 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-02T09:21:45,147 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-02T09:21:45,159 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=7c6d666a4939%2C40561%2C1733131304049.meta, suffix=.meta, logDir=hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/WALs/7c6d666a4939,40561,1733131304049, archiveDir=hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/oldWALs, maxLogs=32 2024-12-02T09:21:45,160 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 7c6d666a4939%2C40561%2C1733131304049.meta.1733131305160.meta 2024-12-02T09:21:45,196 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/WALs/7c6d666a4939,40561,1733131304049/7c6d666a4939%2C40561%2C1733131304049.meta.1733131305160.meta 2024-12-02T09:21:45,200 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46443:46443),(127.0.0.1/127.0.0.1:45285:45285)] 2024-12-02T09:21:45,208 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-02T09:21:45,209 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-02T09:21:45,209 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-02T09:21:45,209 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-02T09:21:45,209 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-02T09:21:45,209 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T09:21:45,209 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-02T09:21:45,209 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-02T09:21:45,215 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T09:21:45,220 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T09:21:45,220 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:21:45,223 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:21:45,223 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T09:21:45,224 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T09:21:45,224 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:21:45,225 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:21:45,225 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T09:21:45,226 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T09:21:45,226 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:21:45,227 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:21:45,227 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T09:21:45,228 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T09:21:45,228 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T09:21:45,231 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T09:21:45,231 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T09:21:45,235 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/data/hbase/meta/1588230740 2024-12-02T09:21:45,236 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/data/hbase/meta/1588230740 2024-12-02T09:21:45,238 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T09:21:45,238 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T09:21:45,239 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-02T09:21:45,240 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T09:21:45,241 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=705416, jitterRate=-0.10301786661148071}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-02T09:21:45,241 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-02T09:21:45,242 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733131305210Writing region info on filesystem at 1733131305210Initializing all the Stores at 1733131305212 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131305212Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131305215 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733131305215Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733131305215Cleaning up temporary data from old regions at 1733131305238 (+23 ms)Running coprocessor post-open hooks at 1733131305241 (+3 ms)Region opened successfully at 1733131305242 (+1 ms) 2024-12-02T09:21:45,243 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733131305126 2024-12-02T09:21:45,255 DEBUG [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-02T09:21:45,255 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=7c6d666a4939,40561,1733131304049 2024-12-02T09:21:45,255 INFO [RS_OPEN_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-02T09:21:45,256 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 7c6d666a4939,40561,1733131304049, state=OPEN 2024-12-02T09:21:45,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x1009a4adad50001, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T09:21:45,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T09:21:45,318 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=7c6d666a4939,40561,1733131304049 2024-12-02T09:21:45,318 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:21:45,318 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T09:21:45,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-02T09:21:45,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=7c6d666a4939,40561,1733131304049 in 345 msec 2024-12-02T09:21:45,324 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-02T09:21:45,324 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 801 msec 2024-12-02T09:21:45,325 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T09:21:45,325 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-02T09:21:45,326 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T09:21:45,326 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c6d666a4939,40561,1733131304049, seqNum=-1] 2024-12-02T09:21:45,327 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T09:21:45,328 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46455, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T09:21:45,332 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 855 msec 2024-12-02T09:21:45,332 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733131305332, completionTime=-1 2024-12-02T09:21:45,332 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-02T09:21:45,332 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T09:21:45,334 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-02T09:21:45,334 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733131365334 2024-12-02T09:21:45,334 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733131425334 2024-12-02T09:21:45,334 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec 2024-12-02T09:21:45,335 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,35331,1733131303873-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T09:21:45,335 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,35331,1733131303873-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:21:45,335 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,35331,1733131303873-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:21:45,335 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-7c6d666a4939:35331, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:21:45,335 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-02T09:21:45,335 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-02T09:21:45,337 DEBUG [master/7c6d666a4939:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-02T09:21:45,339 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.220sec 2024-12-02T09:21:45,339 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-02T09:21:45,339 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-02T09:21:45,340 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-02T09:21:45,340 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-02T09:21:45,340 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-02T09:21:45,340 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,35331,1733131303873-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T09:21:45,340 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,35331,1733131303873-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-02T09:21:45,342 DEBUG [master/7c6d666a4939:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-02T09:21:45,342 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-02T09:21:45,342 INFO [master/7c6d666a4939:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=7c6d666a4939,35331,1733131303873-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T09:21:45,382 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b8e009f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T09:21:45,383 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 7c6d666a4939,35331,-1 for getting cluster id 2024-12-02T09:21:45,383 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T09:21:45,384 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9c7b5800-9400-4147-b096-99af472fc84e' 2024-12-02T09:21:45,384 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T09:21:45,384 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9c7b5800-9400-4147-b096-99af472fc84e" 2024-12-02T09:21:45,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64d87f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T09:21:45,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [7c6d666a4939,35331,-1] 2024-12-02T09:21:45,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T09:21:45,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:21:45,386 INFO [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T09:21:45,387 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@402f2b9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T09:21:45,387 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T09:21:45,388 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=7c6d666a4939,40561,1733131304049, seqNum=-1] 2024-12-02T09:21:45,388 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T09:21:45,389 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T09:21:45,391 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=7c6d666a4939,35331,1733131303873 2024-12-02T09:21:45,391 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T09:21:45,393 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-12-02T09:21:45,393 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-12-02T09:21:45,395 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/WALs/test.com,8080,1, archiveDir=hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/oldWALs, maxLogs=32 2024-12-02T09:21:45,396 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733131305396 2024-12-02T09:21:45,400 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/WALs/test.com,8080,1/test.com%2C8080%2C1.1733131305396 2024-12-02T09:21:45,402 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45285:45285),(127.0.0.1/127.0.0.1:46443:46443)] 2024-12-02T09:21:45,403 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733131305402 2024-12-02T09:21:45,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:45,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:45,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:45,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:45,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:45,408 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/WALs/test.com,8080,1/test.com%2C8080%2C1.1733131305396 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/WALs/test.com,8080,1/test.com%2C8080%2C1.1733131305402 2024-12-02T09:21:45,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36095 is added to blk_1073741835_1011 (size=93) 2024-12-02T09:21:45,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38373 is added to blk_1073741835_1011 (size=93) 2024-12-02T09:21:45,410 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46443:46443),(127.0.0.1/127.0.0.1:45285:45285)] 2024-12-02T09:21:45,415 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/WALs/test.com,8080,1/test.com%2C8080%2C1.1733131305396 to hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/oldWALs/test.com%2C8080%2C1.1733131305396 2024-12-02T09:21:45,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:45,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:45,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:45,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:45,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:45,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38373 is added to blk_1073741836_1012 (size=93) 2024-12-02T09:21:45,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36095 is added to blk_1073741836_1012 (size=93) 2024-12-02T09:21:45,419 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/oldWALs 2024-12-02T09:21:45,419 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1733131305402) 2024-12-02T09:21:45,419 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-02T09:21:45,419 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T09:21:45,419 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T09:21:45,419 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:21:45,419 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:21:45,419 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T09:21:45,419 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-02T09:21:45,420 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1860355745, stopped=false 2024-12-02T09:21:45,420 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=7c6d666a4939,35331,1733131303873 2024-12-02T09:21:45,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T09:21:45,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x1009a4adad50001, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T09:21:45,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:21:45,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x1009a4adad50001, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:21:45,435 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T09:21:45,435 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T09:21:45,436 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T09:21:45,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:21:45,436 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:21:45,436 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40561-0x1009a4adad50001, quorum=127.0.0.1:59473, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T09:21:45,436 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '7c6d666a4939,40561,1733131304049' ***** 2024-12-02T09:21:45,436 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-02T09:21:45,436 INFO [RS:0;7c6d666a4939:40561 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T09:21:45,436 INFO [RS:0;7c6d666a4939:40561 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-02T09:21:45,436 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T09:21:45,436 INFO [RS:0;7c6d666a4939:40561 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-02T09:21:45,436 INFO [RS:0;7c6d666a4939:40561 {}] regionserver.HRegionServer(959): stopping server 7c6d666a4939,40561,1733131304049 2024-12-02T09:21:45,436 INFO [RS:0;7c6d666a4939:40561 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T09:21:45,436 INFO [RS:0;7c6d666a4939:40561 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;7c6d666a4939:40561. 2024-12-02T09:21:45,436 DEBUG [RS:0;7c6d666a4939:40561 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T09:21:45,436 DEBUG [RS:0;7c6d666a4939:40561 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:21:45,436 INFO [RS:0;7c6d666a4939:40561 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T09:21:45,436 INFO [RS:0;7c6d666a4939:40561 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T09:21:45,436 INFO [RS:0;7c6d666a4939:40561 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T09:21:45,436 INFO [RS:0;7c6d666a4939:40561 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-02T09:21:45,437 INFO [RS:0;7c6d666a4939:40561 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-02T09:21:45,437 DEBUG [RS:0;7c6d666a4939:40561 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-02T09:21:45,437 DEBUG [RS:0;7c6d666a4939:40561 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T09:21:45,437 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T09:21:45,437 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T09:21:45,437 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T09:21:45,437 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T09:21:45,437 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T09:21:45,437 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB 2024-12-02T09:21:45,449 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/data/hbase/meta/1588230740/.tmp/ns/3776d33cc8f0407bb13852a93cbcc0f2 is 43, key is default/ns:d/1733131305328/Put/seqid=0 2024-12-02T09:21:45,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38373 is added to blk_1073741837_1013 (size=5153) 2024-12-02T09:21:45,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36095 is added to blk_1073741837_1013 (size=5153) 2024-12-02T09:21:45,453 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/data/hbase/meta/1588230740/.tmp/ns/3776d33cc8f0407bb13852a93cbcc0f2 2024-12-02T09:21:45,457 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/data/hbase/meta/1588230740/.tmp/ns/3776d33cc8f0407bb13852a93cbcc0f2 as hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/data/hbase/meta/1588230740/ns/3776d33cc8f0407bb13852a93cbcc0f2 2024-12-02T09:21:45,464 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/data/hbase/meta/1588230740/ns/3776d33cc8f0407bb13852a93cbcc0f2, entries=2, sequenceid=6, filesize=5.0 K 2024-12-02T09:21:45,465 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 28ms, sequenceid=6, compaction requested=false 2024-12-02T09:21:45,465 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-02T09:21:45,470 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-02T09:21:45,471 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T09:21:45,471 INFO [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T09:21:45,471 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733131305437Running coprocessor pre-close hooks at 1733131305437Disabling compacts and flushes for region at 1733131305437Disabling writes for close at 1733131305437Obtaining lock to block concurrent updates at 1733131305437Preparing flush snapshotting stores in 1588230740 at 1733131305437Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1733131305437Flushing stores of hbase:meta,,1.1588230740 at 1733131305438 (+1 ms)Flushing 1588230740/ns: creating writer at 1733131305438Flushing 1588230740/ns: appending metadata at 1733131305448 (+10 ms)Flushing 1588230740/ns: closing flushed file at 1733131305448Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c9d4cba: reopening flushed file at 1733131305457 (+9 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 28ms, sequenceid=6, compaction requested=false at 1733131305465 (+8 ms)Writing region close event to WAL at 1733131305467 (+2 ms)Running coprocessor post-close hooks at 1733131305471 (+4 ms)Closed at 1733131305471 2024-12-02T09:21:45,471 DEBUG [RS_CLOSE_META-regionserver/7c6d666a4939:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-02T09:21:45,483 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,37839,1733131112142/7c6d666a4939%2C37839%2C1733131112142.meta.1733131113174.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:45,484 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:40751/user/jenkins/test-data/3f92cc35-67be-53b5-d40b-979d9398022c/WALs/7c6d666a4939,39613,1733131113414/7c6d666a4939%2C39613%2C1733131113414.1733131113632 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-12-02T09:21:45,541 INFO [regionserver/7c6d666a4939:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-02T09:21:45,541 INFO [regionserver/7c6d666a4939:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-02T09:21:45,637 INFO [RS:0;7c6d666a4939:40561 {}] regionserver.HRegionServer(976): stopping server 7c6d666a4939,40561,1733131304049; all regions closed. 2024-12-02T09:21:45,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:45,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:45,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:45,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:45,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:45,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38373 is added to blk_1073741834_1010 (size=1152) 2024-12-02T09:21:45,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36095 is added to blk_1073741834_1010 (size=1152) 2024-12-02T09:21:45,643 DEBUG [RS:0;7c6d666a4939:40561 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/oldWALs 2024-12-02T09:21:45,643 INFO [RS:0;7c6d666a4939:40561 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7c6d666a4939%2C40561%2C1733131304049.meta:.meta(num 1733131305160) 2024-12-02T09:21:45,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:45,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:45,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:45,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:45,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:45,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36095 is added to blk_1073741833_1009 (size=93) 2024-12-02T09:21:45,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38373 is added to blk_1073741833_1009 (size=93) 2024-12-02T09:21:45,649 DEBUG [RS:0;7c6d666a4939:40561 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/oldWALs 2024-12-02T09:21:45,649 INFO [RS:0;7c6d666a4939:40561 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 7c6d666a4939%2C40561%2C1733131304049:(num 1733131304674) 2024-12-02T09:21:45,649 DEBUG [RS:0;7c6d666a4939:40561 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T09:21:45,649 INFO [RS:0;7c6d666a4939:40561 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T09:21:45,649 INFO [RS:0;7c6d666a4939:40561 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T09:21:45,649 INFO [RS:0;7c6d666a4939:40561 {}] hbase.ChoreService(370): Chore service for: regionserver/7c6d666a4939:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-02T09:21:45,649 INFO [RS:0;7c6d666a4939:40561 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T09:21:45,649 INFO [regionserver/7c6d666a4939:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T09:21:45,649 INFO [RS:0;7c6d666a4939:40561 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40561 2024-12-02T09:21:45,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T09:21:45,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x1009a4adad50001, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/7c6d666a4939,40561,1733131304049 2024-12-02T09:21:45,686 INFO [RS:0;7c6d666a4939:40561 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T09:21:45,702 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [7c6d666a4939,40561,1733131304049] 2024-12-02T09:21:45,710 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/7c6d666a4939,40561,1733131304049 already deleted, retry=false 2024-12-02T09:21:45,710 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 7c6d666a4939,40561,1733131304049 expired; onlineServers=0 2024-12-02T09:21:45,710 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '7c6d666a4939,35331,1733131303873' ***** 2024-12-02T09:21:45,710 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-02T09:21:45,710 INFO [M:0;7c6d666a4939:35331 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T09:21:45,710 INFO [M:0;7c6d666a4939:35331 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T09:21:45,710 DEBUG [M:0;7c6d666a4939:35331 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-02T09:21:45,710 DEBUG [M:0;7c6d666a4939:35331 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-02T09:21:45,710 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-02T09:21:45,710 DEBUG [master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.small.0-1733131304492 {}] cleaner.HFileCleaner(306): Exit Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.small.0-1733131304492,5,FailOnTimeoutGroup] 2024-12-02T09:21:45,710 DEBUG [master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.large.0-1733131304491 {}] cleaner.HFileCleaner(306): Exit Thread[master/7c6d666a4939:0:becomeActiveMaster-HFileCleaner.large.0-1733131304491,5,FailOnTimeoutGroup] 2024-12-02T09:21:45,711 INFO [M:0;7c6d666a4939:35331 {}] hbase.ChoreService(370): Chore service for: master/7c6d666a4939:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-02T09:21:45,711 INFO [M:0;7c6d666a4939:35331 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T09:21:45,711 DEBUG [M:0;7c6d666a4939:35331 {}] master.HMaster(1795): Stopping service threads 2024-12-02T09:21:45,711 INFO [M:0;7c6d666a4939:35331 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-02T09:21:45,711 INFO [M:0;7c6d666a4939:35331 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T09:21:45,711 INFO [M:0;7c6d666a4939:35331 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T09:21:45,711 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-02T09:21:45,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-02T09:21:45,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T09:21:45,718 DEBUG [M:0;7c6d666a4939:35331 {}] zookeeper.ZKUtil(347): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-02T09:21:45,718 WARN [M:0;7c6d666a4939:35331 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-02T09:21:45,719 INFO [M:0;7c6d666a4939:35331 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/.lastflushedseqids 2024-12-02T09:21:45,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38373 is added to blk_1073741838_1014 (size=99) 2024-12-02T09:21:45,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36095 is added to blk_1073741838_1014 (size=99) 2024-12-02T09:21:45,725 INFO [M:0;7c6d666a4939:35331 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T09:21:45,725 INFO [M:0;7c6d666a4939:35331 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-02T09:21:45,725 DEBUG [M:0;7c6d666a4939:35331 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T09:21:45,725 INFO [M:0;7c6d666a4939:35331 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:21:45,725 DEBUG [M:0;7c6d666a4939:35331 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:21:45,725 DEBUG [M:0;7c6d666a4939:35331 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T09:21:45,725 DEBUG [M:0;7c6d666a4939:35331 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:21:45,725 INFO [M:0;7c6d666a4939:35331 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB 2024-12-02T09:21:45,738 DEBUG [M:0;7c6d666a4939:35331 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3e8c6ecb3fcf4cbab4457e3bccc08e13 is 82, key is hbase:meta,,1/info:regioninfo/1733131305255/Put/seqid=0 2024-12-02T09:21:45,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36095 is added to blk_1073741839_1015 (size=5672) 2024-12-02T09:21:45,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38373 is added to blk_1073741839_1015 (size=5672) 2024-12-02T09:21:45,743 INFO [M:0;7c6d666a4939:35331 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3e8c6ecb3fcf4cbab4457e3bccc08e13 2024-12-02T09:21:45,760 DEBUG [M:0;7c6d666a4939:35331 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a6c43623892c4655852803686f815cb2 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1733131305332/Put/seqid=0 2024-12-02T09:21:45,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36095 is added to blk_1073741840_1016 (size=5275) 2024-12-02T09:21:45,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38373 is added to blk_1073741840_1016 (size=5275) 2024-12-02T09:21:45,764 INFO [M:0;7c6d666a4939:35331 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a6c43623892c4655852803686f815cb2 2024-12-02T09:21:45,781 DEBUG [M:0;7c6d666a4939:35331 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dd054379486141d8959a8973ba78a807 is 69, key is 7c6d666a4939,40561,1733131304049/rs:state/1733131304523/Put/seqid=0 2024-12-02T09:21:45,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36095 is added to blk_1073741841_1017 (size=5156) 2024-12-02T09:21:45,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38373 is added to blk_1073741841_1017 (size=5156) 2024-12-02T09:21:45,786 INFO [M:0;7c6d666a4939:35331 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dd054379486141d8959a8973ba78a807 2024-12-02T09:21:45,801 DEBUG [M:0;7c6d666a4939:35331 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6331b4fc336c40b59d388914b7ff83e8 is 52, key is load_balancer_on/state:d/1733131305392/Put/seqid=0 2024-12-02T09:21:45,802 INFO [RS:0;7c6d666a4939:40561 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T09:21:45,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x1009a4adad50001, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:21:45,802 INFO [RS:0;7c6d666a4939:40561 {}] regionserver.HRegionServer(1031): Exiting; stopping=7c6d666a4939,40561,1733131304049; zookeeper connection closed. 2024-12-02T09:21:45,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40561-0x1009a4adad50001, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:21:45,803 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@483efe8e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@483efe8e 2024-12-02T09:21:45,803 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-02T09:21:45,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38373 is added to blk_1073741842_1018 (size=5056) 2024-12-02T09:21:45,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36095 is added to blk_1073741842_1018 (size=5056) 2024-12-02T09:21:45,806 INFO [M:0;7c6d666a4939:35331 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6331b4fc336c40b59d388914b7ff83e8 2024-12-02T09:21:45,810 DEBUG [M:0;7c6d666a4939:35331 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3e8c6ecb3fcf4cbab4457e3bccc08e13 as hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3e8c6ecb3fcf4cbab4457e3bccc08e13 2024-12-02T09:21:45,814 INFO [M:0;7c6d666a4939:35331 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3e8c6ecb3fcf4cbab4457e3bccc08e13, entries=8, sequenceid=29, filesize=5.5 K 2024-12-02T09:21:45,815 DEBUG [M:0;7c6d666a4939:35331 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a6c43623892c4655852803686f815cb2 as hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a6c43623892c4655852803686f815cb2 2024-12-02T09:21:45,819 INFO [M:0;7c6d666a4939:35331 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a6c43623892c4655852803686f815cb2, entries=3, sequenceid=29, filesize=5.2 K 2024-12-02T09:21:45,820 DEBUG [M:0;7c6d666a4939:35331 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dd054379486141d8959a8973ba78a807 as hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/dd054379486141d8959a8973ba78a807 2024-12-02T09:21:45,824 INFO [M:0;7c6d666a4939:35331 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/dd054379486141d8959a8973ba78a807, entries=1, sequenceid=29, filesize=5.0 K 2024-12-02T09:21:45,825 DEBUG [M:0;7c6d666a4939:35331 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6331b4fc336c40b59d388914b7ff83e8 as hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6331b4fc336c40b59d388914b7ff83e8 2024-12-02T09:21:45,829 INFO [M:0;7c6d666a4939:35331 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44613/user/jenkins/test-data/cd21b13d-35e4-4190-b196-fda11ea0d5a8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6331b4fc336c40b59d388914b7ff83e8, entries=1, sequenceid=29, filesize=4.9 K 2024-12-02T09:21:45,830 INFO [M:0;7c6d666a4939:35331 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 104ms, sequenceid=29, compaction requested=false 2024-12-02T09:21:45,831 INFO [M:0;7c6d666a4939:35331 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T09:21:45,831 DEBUG [M:0;7c6d666a4939:35331 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733131305725Disabling compacts and flushes for region at 1733131305725Disabling writes for close at 1733131305725Obtaining lock to block concurrent updates at 1733131305725Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733131305725Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1733131305726 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733131305727 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733131305727Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733131305738 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733131305738Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733131305747 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733131305760 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733131305760Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733131305767 (+7 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733131305781 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733131305781Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733131305790 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733131305801 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733131305801Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@207cb432: reopening flushed file at 1733131305809 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25888ab1: reopening flushed file at 1733131305814 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3de35fa2: reopening flushed file at 1733131305819 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7790440c: reopening flushed file at 1733131305824 (+5 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 104ms, sequenceid=29, compaction requested=false at 1733131305830 (+6 ms)Writing region close event to WAL at 1733131305830Closed at 1733131305830 2024-12-02T09:21:45,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:45,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:45,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:45,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:45,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-02T09:21:45,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38373 is added to blk_1073741830_1006 (size=10311) 2024-12-02T09:21:45,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36095 is added to blk_1073741830_1006 (size=10311) 2024-12-02T09:21:45,833 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T09:21:45,833 INFO [M:0;7c6d666a4939:35331 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-02T09:21:45,833 INFO [M:0;7c6d666a4939:35331 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35331 2024-12-02T09:21:45,833 INFO [M:0;7c6d666a4939:35331 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T09:21:45,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:21:45,944 INFO [M:0;7c6d666a4939:35331 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T09:21:45,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35331-0x1009a4adad50000, quorum=127.0.0.1:59473, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T09:21:45,946 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@335e1da0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:21:45,946 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@9279e7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:21:45,946 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:21:45,946 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@f9db876{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:21:45,947 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1cfa2b02{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/hadoop.log.dir/,STOPPED} 2024-12-02T09:21:45,948 WARN [BP-1838583546-172.17.0.3-1733131302118 heartbeating to localhost/127.0.0.1:44613 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T09:21:45,948 WARN [BP-1838583546-172.17.0.3-1733131302118 heartbeating to localhost/127.0.0.1:44613 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1838583546-172.17.0.3-1733131302118 (Datanode Uuid e66fb798-a919-4627-bb84-7b637111a807) service to localhost/127.0.0.1:44613 2024-12-02T09:21:45,948 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T09:21:45,948 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T09:21:45,948 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/cluster_62bbdbaa-2ad6-42c0-7b1c-03d67d162a57/data/data3/current/BP-1838583546-172.17.0.3-1733131302118 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:21:45,949 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/cluster_62bbdbaa-2ad6-42c0-7b1c-03d67d162a57/data/data4/current/BP-1838583546-172.17.0.3-1733131302118 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:21:45,949 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T09:21:45,951 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1efd209{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T09:21:45,951 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@49e6dd92{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:21:45,951 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:21:45,951 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7acee9f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:21:45,951 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@624ed4c3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/hadoop.log.dir/,STOPPED} 2024-12-02T09:21:45,953 WARN [BP-1838583546-172.17.0.3-1733131302118 heartbeating to localhost/127.0.0.1:44613 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T09:21:45,953 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T09:21:45,953 WARN [BP-1838583546-172.17.0.3-1733131302118 heartbeating to localhost/127.0.0.1:44613 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1838583546-172.17.0.3-1733131302118 (Datanode Uuid 6a262b8d-8647-4eaa-a35d-a3d243983b6b) service to localhost/127.0.0.1:44613 2024-12-02T09:21:45,953 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T09:21:45,953 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/cluster_62bbdbaa-2ad6-42c0-7b1c-03d67d162a57/data/data1/current/BP-1838583546-172.17.0.3-1733131302118 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:21:45,954 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/cluster_62bbdbaa-2ad6-42c0-7b1c-03d67d162a57/data/data2/current/BP-1838583546-172.17.0.3-1733131302118 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T09:21:45,954 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T09:21:45,960 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7ffc1e9{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T09:21:45,960 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5e2fef96{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T09:21:45,960 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T09:21:45,961 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@512a083f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T09:21:45,961 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@19f4d4a5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0a7823f7-6583-6d64-1faa-5e88b095e85f/hadoop.log.dir/,STOPPED} 2024-12-02T09:21:45,967 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-02T09:21:45,984 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-02T09:21:45,993 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=268 (was 230) Potentially hanging thread: nioEventLoopGroup-42-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:44613 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-42-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-44-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-43-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44613 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-44-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-45-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44613 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: regionserver/7c6d666a4939:0.leaseChecker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82) Potentially hanging thread: nioEventLoopGroup-43-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:44613 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-43-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-42-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44613 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-45-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-45-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44613 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44613 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44613 from jenkins.hfs.7 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-44-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=541 (was 518) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=227 (was 227), ProcessCount=11 (was 11), AvailableMemoryMB=393 (was 451)